1 /* $OpenBSD: if_mcx.c,v 1.106 2022/11/22 06:48:32 jmatthew Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 #include "vlan.h" 22 #include "kstat.h" 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/kernel.h> 29 #include <sys/socket.h> 30 #include <sys/device.h> 31 #include <sys/pool.h> 32 #include <sys/queue.h> 33 #include <sys/timeout.h> 34 #include <sys/task.h> 35 #include <sys/atomic.h> 36 #include <sys/timetc.h> 37 #include <sys/intrmap.h> 38 39 #include <machine/bus.h> 40 #include <machine/intr.h> 41 42 #include <net/if.h> 43 #include <net/if_dl.h> 44 #include <net/if_media.h> 45 #include <net/toeplitz.h> 46 47 #if NBPFILTER > 0 48 #include <net/bpf.h> 49 #endif 50 51 #if NKSTAT > 0 52 #include <sys/kstat.h> 53 #endif 54 55 #include <netinet/in.h> 56 #include <netinet/if_ether.h> 57 58 #include <dev/pci/pcireg.h> 59 #include <dev/pci/pcivar.h> 60 #include <dev/pci/pcidevs.h> 61 62 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 63 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 64 65 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 66 67 #define MCX_FW_VER 0x0000 68 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 69 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 70 #define MCX_CMDIF_FW_SUBVER 0x0004 71 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 72 #define MCX_CMDIF(_v) ((_v) >> 16) 73 74 #define MCX_ISSI 1 /* as per the PRM */ 75 #define MCX_CMD_IF_SUPPORTED 5 76 77 #define MCX_HARDMTU 9500 78 79 enum mcx_cmdq_slot { 80 MCX_CMDQ_SLOT_POLL = 0, 81 MCX_CMDQ_SLOT_IOCTL, 82 MCX_CMDQ_SLOT_KSTAT, 83 MCX_CMDQ_SLOT_LINK, 84 85 MCX_CMDQ_NUM_SLOTS 86 }; 87 88 #define MCX_PAGE_SHIFT 12 89 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 90 91 /* queue sizes */ 92 #define MCX_LOG_EQ_SIZE 7 93 #define MCX_LOG_CQ_SIZE 12 94 #define MCX_LOG_RQ_SIZE 10 95 #define MCX_LOG_SQ_SIZE 11 96 97 #define MCX_MAX_QUEUES 16 98 99 /* completion event moderation - about 10khz, or 90% of the cq */ 100 #define MCX_CQ_MOD_PERIOD 50 101 #define MCX_CQ_MOD_COUNTER \ 102 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 103 104 #define MCX_LOG_SQ_ENTRY_SIZE 6 105 #define MCX_SQ_ENTRY_MAX_SLOTS 4 106 #define MCX_SQ_SEGS_PER_SLOT \ 107 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 108 #define MCX_SQ_MAX_SEGMENTS \ 109 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 110 111 #define MCX_LOG_FLOW_TABLE_SIZE 5 112 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 113 #define MCX_NUM_MCAST_FLOWS \ 114 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 115 116 #define MCX_SQ_INLINE_SIZE 18 117 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 118 119 /* doorbell offsets */ 120 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 121 122 #define MCX_CQ_DOORBELL_BASE 0 123 #define MCX_CQ_DOORBELL_STRIDE 64 124 125 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 126 #define MCX_WQ_DOORBELL_STRIDE 64 127 /* make sure the doorbells fit */ 128 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 129 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 130 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 131 132 #define MCX_WQ_DOORBELL_MASK 0xffff 133 134 /* uar registers */ 135 #define MCX_UAR_CQ_DOORBELL 0x20 136 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 137 #define MCX_UAR_EQ_DOORBELL 0x48 138 #define MCX_UAR_BF 0x800 139 140 #define MCX_CMDQ_ADDR_HI 0x0010 141 #define MCX_CMDQ_ADDR_LO 0x0014 142 #define MCX_CMDQ_ADDR_NMASK 0xfff 143 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 144 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 145 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 146 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 147 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 148 149 #define MCX_CMDQ_DOORBELL 0x0018 150 151 #define MCX_STATE 0x01fc 152 #define MCX_STATE_MASK (1 << 31) 153 #define MCX_STATE_INITIALIZING (1 << 31) 154 #define MCX_STATE_READY (0 << 31) 155 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 156 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 157 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 158 159 #define MCX_INTERNAL_TIMER 0x1000 160 #define MCX_INTERNAL_TIMER_H 0x1000 161 #define MCX_INTERNAL_TIMER_L 0x1004 162 163 #define MCX_CLEAR_INT 0x100c 164 165 #define MCX_REG_OP_WRITE 0 166 #define MCX_REG_OP_READ 1 167 168 #define MCX_REG_PMLP 0x5002 169 #define MCX_REG_PMTU 0x5003 170 #define MCX_REG_PTYS 0x5004 171 #define MCX_REG_PAOS 0x5006 172 #define MCX_REG_PFCC 0x5007 173 #define MCX_REG_PPCNT 0x5008 174 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 175 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 176 #define MCX_REG_MCIA 0x9014 177 #define MCX_REG_MCAM 0x907f 178 179 #define MCX_ETHER_CAP_SGMII 0 180 #define MCX_ETHER_CAP_1000_KX 1 181 #define MCX_ETHER_CAP_10G_CX4 2 182 #define MCX_ETHER_CAP_10G_KX4 3 183 #define MCX_ETHER_CAP_10G_KR 4 184 #define MCX_ETHER_CAP_40G_CR4 6 185 #define MCX_ETHER_CAP_40G_KR4 7 186 #define MCX_ETHER_CAP_10G_CR 12 187 #define MCX_ETHER_CAP_10G_SR 13 188 #define MCX_ETHER_CAP_10G_LR 14 189 #define MCX_ETHER_CAP_40G_SR4 15 190 #define MCX_ETHER_CAP_40G_LR4 16 191 #define MCX_ETHER_CAP_50G_SR2 18 192 #define MCX_ETHER_CAP_100G_CR4 20 193 #define MCX_ETHER_CAP_100G_SR4 21 194 #define MCX_ETHER_CAP_100G_KR4 22 195 #define MCX_ETHER_CAP_25G_CR 27 196 #define MCX_ETHER_CAP_25G_KR 28 197 #define MCX_ETHER_CAP_25G_SR 29 198 #define MCX_ETHER_CAP_50G_CR2 30 199 #define MCX_ETHER_CAP_50G_KR2 31 200 201 #define MCX_MAX_CQE 32 202 203 #define MCX_CMD_QUERY_HCA_CAP 0x100 204 #define MCX_CMD_QUERY_ADAPTER 0x101 205 #define MCX_CMD_INIT_HCA 0x102 206 #define MCX_CMD_TEARDOWN_HCA 0x103 207 #define MCX_CMD_ENABLE_HCA 0x104 208 #define MCX_CMD_DISABLE_HCA 0x105 209 #define MCX_CMD_QUERY_PAGES 0x107 210 #define MCX_CMD_MANAGE_PAGES 0x108 211 #define MCX_CMD_SET_HCA_CAP 0x109 212 #define MCX_CMD_QUERY_ISSI 0x10a 213 #define MCX_CMD_SET_ISSI 0x10b 214 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 215 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 216 #define MCX_CMD_CREATE_EQ 0x301 217 #define MCX_CMD_DESTROY_EQ 0x302 218 #define MCX_CMD_QUERY_EQ 0x303 219 #define MCX_CMD_CREATE_CQ 0x400 220 #define MCX_CMD_DESTROY_CQ 0x401 221 #define MCX_CMD_QUERY_CQ 0x402 222 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 223 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 224 0x755 225 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 226 #define MCX_CMD_ALLOC_PD 0x800 227 #define MCX_CMD_ALLOC_UAR 0x802 228 #define MCX_CMD_ACCESS_REG 0x805 229 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 230 #define MCX_CMD_CREATE_TIR 0x900 231 #define MCX_CMD_DESTROY_TIR 0x902 232 #define MCX_CMD_CREATE_SQ 0x904 233 #define MCX_CMD_MODIFY_SQ 0x905 234 #define MCX_CMD_DESTROY_SQ 0x906 235 #define MCX_CMD_QUERY_SQ 0x907 236 #define MCX_CMD_CREATE_RQ 0x908 237 #define MCX_CMD_MODIFY_RQ 0x909 238 #define MCX_CMD_DESTROY_RQ 0x90a 239 #define MCX_CMD_QUERY_RQ 0x90b 240 #define MCX_CMD_CREATE_TIS 0x912 241 #define MCX_CMD_DESTROY_TIS 0x914 242 #define MCX_CMD_CREATE_RQT 0x916 243 #define MCX_CMD_DESTROY_RQT 0x918 244 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 245 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 246 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 247 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 248 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 249 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 250 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 251 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 252 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 253 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 254 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 255 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 256 257 #define MCX_QUEUE_STATE_RST 0 258 #define MCX_QUEUE_STATE_RDY 1 259 #define MCX_QUEUE_STATE_ERR 3 260 261 #define MCX_FLOW_TABLE_TYPE_RX 0 262 #define MCX_FLOW_TABLE_TYPE_TX 1 263 264 #define MCX_CMDQ_INLINE_DATASIZE 16 265 266 struct mcx_cmdq_entry { 267 uint8_t cq_type; 268 #define MCX_CMDQ_TYPE_PCIE 0x7 269 uint8_t cq_reserved0[3]; 270 271 uint32_t cq_input_length; 272 uint64_t cq_input_ptr; 273 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 274 275 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 276 uint64_t cq_output_ptr; 277 uint32_t cq_output_length; 278 279 uint8_t cq_token; 280 uint8_t cq_signature; 281 uint8_t cq_reserved1[1]; 282 uint8_t cq_status; 283 #define MCX_CQ_STATUS_SHIFT 1 284 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 285 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 286 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 287 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 288 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 289 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 290 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 291 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 292 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 293 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 294 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 295 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 296 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 297 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 298 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 299 (0x10 << MCX_CQ_STATUS_SHIFT) 300 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 301 #define MCX_CQ_STATUS_OWN_MASK 0x1 302 #define MCX_CQ_STATUS_OWN_SW 0x0 303 #define MCX_CQ_STATUS_OWN_HW 0x1 304 } __packed __aligned(8); 305 306 #define MCX_CMDQ_MAILBOX_DATASIZE 512 307 308 struct mcx_cmdq_mailbox { 309 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 310 uint8_t mb_reserved0[48]; 311 uint64_t mb_next_ptr; 312 uint32_t mb_block_number; 313 uint8_t mb_reserved1[1]; 314 uint8_t mb_token; 315 uint8_t mb_ctrl_signature; 316 uint8_t mb_signature; 317 } __packed __aligned(8); 318 319 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 320 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 321 MCX_CMDQ_MAILBOX_ALIGN) 322 /* 323 * command mailbox structures 324 */ 325 326 struct mcx_cmd_enable_hca_in { 327 uint16_t cmd_opcode; 328 uint8_t cmd_reserved0[4]; 329 uint16_t cmd_op_mod; 330 uint8_t cmd_reserved1[2]; 331 uint16_t cmd_function_id; 332 uint8_t cmd_reserved2[4]; 333 } __packed __aligned(4); 334 335 struct mcx_cmd_enable_hca_out { 336 uint8_t cmd_status; 337 uint8_t cmd_reserved0[3]; 338 uint32_t cmd_syndrome; 339 uint8_t cmd_reserved1[4]; 340 } __packed __aligned(4); 341 342 struct mcx_cmd_init_hca_in { 343 uint16_t cmd_opcode; 344 uint8_t cmd_reserved0[4]; 345 uint16_t cmd_op_mod; 346 uint8_t cmd_reserved1[8]; 347 } __packed __aligned(4); 348 349 struct mcx_cmd_init_hca_out { 350 uint8_t cmd_status; 351 uint8_t cmd_reserved0[3]; 352 uint32_t cmd_syndrome; 353 uint8_t cmd_reserved1[8]; 354 } __packed __aligned(4); 355 356 struct mcx_cmd_teardown_hca_in { 357 uint16_t cmd_opcode; 358 uint8_t cmd_reserved0[4]; 359 uint16_t cmd_op_mod; 360 uint8_t cmd_reserved1[2]; 361 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 362 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 363 uint16_t cmd_profile; 364 uint8_t cmd_reserved2[4]; 365 } __packed __aligned(4); 366 367 struct mcx_cmd_teardown_hca_out { 368 uint8_t cmd_status; 369 uint8_t cmd_reserved0[3]; 370 uint32_t cmd_syndrome; 371 uint8_t cmd_reserved1[8]; 372 } __packed __aligned(4); 373 374 struct mcx_cmd_access_reg_in { 375 uint16_t cmd_opcode; 376 uint8_t cmd_reserved0[4]; 377 uint16_t cmd_op_mod; 378 uint8_t cmd_reserved1[2]; 379 uint16_t cmd_register_id; 380 uint32_t cmd_argument; 381 } __packed __aligned(4); 382 383 struct mcx_cmd_access_reg_out { 384 uint8_t cmd_status; 385 uint8_t cmd_reserved0[3]; 386 uint32_t cmd_syndrome; 387 uint8_t cmd_reserved1[8]; 388 } __packed __aligned(4); 389 390 struct mcx_reg_pmtu { 391 uint8_t rp_reserved1; 392 uint8_t rp_local_port; 393 uint8_t rp_reserved2[2]; 394 uint16_t rp_max_mtu; 395 uint8_t rp_reserved3[2]; 396 uint16_t rp_admin_mtu; 397 uint8_t rp_reserved4[2]; 398 uint16_t rp_oper_mtu; 399 uint8_t rp_reserved5[2]; 400 } __packed __aligned(4); 401 402 struct mcx_reg_ptys { 403 uint8_t rp_reserved1; 404 uint8_t rp_local_port; 405 uint8_t rp_reserved2; 406 uint8_t rp_proto_mask; 407 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 408 uint8_t rp_reserved3[8]; 409 uint32_t rp_eth_proto_cap; 410 uint8_t rp_reserved4[8]; 411 uint32_t rp_eth_proto_admin; 412 uint8_t rp_reserved5[8]; 413 uint32_t rp_eth_proto_oper; 414 uint8_t rp_reserved6[24]; 415 } __packed __aligned(4); 416 417 struct mcx_reg_paos { 418 uint8_t rp_reserved1; 419 uint8_t rp_local_port; 420 uint8_t rp_admin_status; 421 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 422 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 423 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 424 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 425 uint8_t rp_oper_status; 426 #define MCX_REG_PAOS_OPER_STATUS_UP 1 427 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 428 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 429 uint8_t rp_admin_state_update; 430 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 431 uint8_t rp_reserved2[11]; 432 } __packed __aligned(4); 433 434 struct mcx_reg_pfcc { 435 uint8_t rp_reserved1; 436 uint8_t rp_local_port; 437 uint8_t rp_reserved2[3]; 438 uint8_t rp_prio_mask_tx; 439 uint8_t rp_reserved3; 440 uint8_t rp_prio_mask_rx; 441 uint8_t rp_pptx_aptx; 442 uint8_t rp_pfctx; 443 uint8_t rp_fctx_dis; 444 uint8_t rp_reserved4; 445 uint8_t rp_pprx_aprx; 446 uint8_t rp_pfcrx; 447 uint8_t rp_reserved5[2]; 448 uint16_t rp_dev_stall_min; 449 uint16_t rp_dev_stall_crit; 450 uint8_t rp_reserved6[12]; 451 } __packed __aligned(4); 452 453 #define MCX_PMLP_MODULE_NUM_MASK 0xff 454 struct mcx_reg_pmlp { 455 uint8_t rp_rxtx; 456 uint8_t rp_local_port; 457 uint8_t rp_reserved0; 458 uint8_t rp_width; 459 uint32_t rp_lane0_mapping; 460 uint32_t rp_lane1_mapping; 461 uint32_t rp_lane2_mapping; 462 uint32_t rp_lane3_mapping; 463 uint8_t rp_reserved1[44]; 464 } __packed __aligned(4); 465 466 struct mcx_reg_ppcnt { 467 uint8_t ppcnt_swid; 468 uint8_t ppcnt_local_port; 469 uint8_t ppcnt_pnat; 470 uint8_t ppcnt_grp; 471 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 472 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 473 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 474 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 475 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 476 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 477 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 478 479 uint8_t ppcnt_clr; 480 uint8_t ppcnt_reserved1[2]; 481 uint8_t ppcnt_prio_tc; 482 #define MCX_REG_PPCNT_CLR (1 << 7) 483 484 uint8_t ppcnt_counter_set[248]; 485 } __packed __aligned(8); 486 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 487 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 488 sizeof(uint64_t)) == 0); 489 490 enum mcx_ppcnt_ieee8023 { 491 frames_transmitted_ok, 492 frames_received_ok, 493 frame_check_sequence_errors, 494 alignment_errors, 495 octets_transmitted_ok, 496 octets_received_ok, 497 multicast_frames_xmitted_ok, 498 broadcast_frames_xmitted_ok, 499 multicast_frames_received_ok, 500 broadcast_frames_received_ok, 501 in_range_length_errors, 502 out_of_range_length_field, 503 frame_too_long_errors, 504 symbol_error_during_carrier, 505 mac_control_frames_transmitted, 506 mac_control_frames_received, 507 unsupported_opcodes_received, 508 pause_mac_ctrl_frames_received, 509 pause_mac_ctrl_frames_transmitted, 510 511 mcx_ppcnt_ieee8023_count 512 }; 513 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 514 515 enum mcx_ppcnt_rfc2863 { 516 in_octets, 517 in_ucast_pkts, 518 in_discards, 519 in_errors, 520 in_unknown_protos, 521 out_octets, 522 out_ucast_pkts, 523 out_discards, 524 out_errors, 525 in_multicast_pkts, 526 in_broadcast_pkts, 527 out_multicast_pkts, 528 out_broadcast_pkts, 529 530 mcx_ppcnt_rfc2863_count 531 }; 532 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 533 534 enum mcx_ppcnt_rfc2819 { 535 drop_events, 536 octets, 537 pkts, 538 broadcast_pkts, 539 multicast_pkts, 540 crc_align_errors, 541 undersize_pkts, 542 oversize_pkts, 543 fragments, 544 jabbers, 545 collisions, 546 pkts64octets, 547 pkts65to127octets, 548 pkts128to255octets, 549 pkts256to511octets, 550 pkts512to1023octets, 551 pkts1024to1518octets, 552 pkts1519to2047octets, 553 pkts2048to4095octets, 554 pkts4096to8191octets, 555 pkts8192to10239octets, 556 557 mcx_ppcnt_rfc2819_count 558 }; 559 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 560 561 enum mcx_ppcnt_rfc3635 { 562 dot3stats_alignment_errors, 563 dot3stats_fcs_errors, 564 dot3stats_single_collision_frames, 565 dot3stats_multiple_collision_frames, 566 dot3stats_sqe_test_errors, 567 dot3stats_deferred_transmissions, 568 dot3stats_late_collisions, 569 dot3stats_excessive_collisions, 570 dot3stats_internal_mac_transmit_errors, 571 dot3stats_carrier_sense_errors, 572 dot3stats_frame_too_longs, 573 dot3stats_internal_mac_receive_errors, 574 dot3stats_symbol_errors, 575 dot3control_in_unknown_opcodes, 576 dot3in_pause_frames, 577 dot3out_pause_frames, 578 579 mcx_ppcnt_rfc3635_count 580 }; 581 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 582 583 struct mcx_reg_mcam { 584 uint8_t _reserved1[1]; 585 uint8_t mcam_feature_group; 586 uint8_t _reserved2[1]; 587 uint8_t mcam_access_reg_group; 588 uint8_t _reserved3[4]; 589 uint8_t mcam_access_reg_cap_mask[16]; 590 uint8_t _reserved4[16]; 591 uint8_t mcam_feature_cap_mask[16]; 592 uint8_t _reserved5[16]; 593 } __packed __aligned(4); 594 595 #define MCX_BITFIELD_BIT(bf, b) (bf[(sizeof bf - 1) - (b / 8)] & (b % 8)) 596 597 #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP 6 598 599 struct mcx_reg_mtcap { 600 uint8_t _reserved1[3]; 601 uint8_t mtcap_sensor_count; 602 uint8_t _reserved2[4]; 603 604 uint64_t mtcap_sensor_map; 605 }; 606 607 struct mcx_reg_mtmp { 608 uint8_t _reserved1[2]; 609 uint16_t mtmp_sensor_index; 610 611 uint8_t _reserved2[2]; 612 uint16_t mtmp_temperature; 613 614 uint16_t mtmp_mte_mtr; 615 #define MCX_REG_MTMP_MTE (1 << 15) 616 #define MCX_REG_MTMP_MTR (1 << 14) 617 uint16_t mtmp_max_temperature; 618 619 uint16_t mtmp_tee; 620 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 621 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 622 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 623 uint16_t mtmp_temperature_threshold_hi; 624 625 uint8_t _reserved3[2]; 626 uint16_t mtmp_temperature_threshold_lo; 627 628 uint8_t _reserved4[4]; 629 630 uint8_t mtmp_sensor_name[8]; 631 }; 632 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 633 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 634 635 #define MCX_MCIA_EEPROM_BYTES 32 636 struct mcx_reg_mcia { 637 uint8_t rm_l; 638 uint8_t rm_module; 639 uint8_t rm_reserved0; 640 uint8_t rm_status; 641 uint8_t rm_i2c_addr; 642 uint8_t rm_page_num; 643 uint16_t rm_dev_addr; 644 uint16_t rm_reserved1; 645 uint16_t rm_size; 646 uint32_t rm_reserved2; 647 uint8_t rm_data[48]; 648 } __packed __aligned(4); 649 650 struct mcx_cmd_query_issi_in { 651 uint16_t cmd_opcode; 652 uint8_t cmd_reserved0[4]; 653 uint16_t cmd_op_mod; 654 uint8_t cmd_reserved1[8]; 655 } __packed __aligned(4); 656 657 struct mcx_cmd_query_issi_il_out { 658 uint8_t cmd_status; 659 uint8_t cmd_reserved0[3]; 660 uint32_t cmd_syndrome; 661 uint8_t cmd_reserved1[2]; 662 uint16_t cmd_current_issi; 663 uint8_t cmd_reserved2[4]; 664 } __packed __aligned(4); 665 666 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 667 668 struct mcx_cmd_query_issi_mb_out { 669 uint8_t cmd_reserved2[16]; 670 uint8_t cmd_supported_issi[80]; /* very big endian */ 671 } __packed __aligned(4); 672 673 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 674 675 struct mcx_cmd_set_issi_in { 676 uint16_t cmd_opcode; 677 uint8_t cmd_reserved0[4]; 678 uint16_t cmd_op_mod; 679 uint8_t cmd_reserved1[2]; 680 uint16_t cmd_current_issi; 681 uint8_t cmd_reserved2[4]; 682 } __packed __aligned(4); 683 684 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 685 686 struct mcx_cmd_set_issi_out { 687 uint8_t cmd_status; 688 uint8_t cmd_reserved0[3]; 689 uint32_t cmd_syndrome; 690 uint8_t cmd_reserved1[8]; 691 } __packed __aligned(4); 692 693 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 694 695 struct mcx_cmd_query_pages_in { 696 uint16_t cmd_opcode; 697 uint8_t cmd_reserved0[4]; 698 uint16_t cmd_op_mod; 699 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 700 #define MCX_CMD_QUERY_PAGES_INIT 0x02 701 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 702 uint8_t cmd_reserved1[8]; 703 } __packed __aligned(4); 704 705 struct mcx_cmd_query_pages_out { 706 uint8_t cmd_status; 707 uint8_t cmd_reserved0[3]; 708 uint32_t cmd_syndrome; 709 uint8_t cmd_reserved1[2]; 710 uint16_t cmd_func_id; 711 int32_t cmd_num_pages; 712 } __packed __aligned(4); 713 714 struct mcx_cmd_manage_pages_in { 715 uint16_t cmd_opcode; 716 uint8_t cmd_reserved0[4]; 717 uint16_t cmd_op_mod; 718 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 719 0x00 720 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 721 0x01 722 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 723 0x02 724 uint8_t cmd_reserved1[2]; 725 uint16_t cmd_func_id; 726 uint32_t cmd_input_num_entries; 727 } __packed __aligned(4); 728 729 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 730 731 struct mcx_cmd_manage_pages_out { 732 uint8_t cmd_status; 733 uint8_t cmd_reserved0[3]; 734 uint32_t cmd_syndrome; 735 uint32_t cmd_output_num_entries; 736 uint8_t cmd_reserved1[4]; 737 } __packed __aligned(4); 738 739 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 740 741 struct mcx_cmd_query_hca_cap_in { 742 uint16_t cmd_opcode; 743 uint8_t cmd_reserved0[4]; 744 uint16_t cmd_op_mod; 745 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 746 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 747 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 748 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 749 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 750 uint8_t cmd_reserved1[8]; 751 } __packed __aligned(4); 752 753 struct mcx_cmd_query_hca_cap_out { 754 uint8_t cmd_status; 755 uint8_t cmd_reserved0[3]; 756 uint32_t cmd_syndrome; 757 uint8_t cmd_reserved1[8]; 758 } __packed __aligned(4); 759 760 #define MCX_HCA_CAP_LEN 0x1000 761 #define MCX_HCA_CAP_NMAILBOXES \ 762 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 763 764 #if __GNUC_PREREQ__(4, 3) 765 #define __counter__ __COUNTER__ 766 #else 767 #define __counter__ __LINE__ 768 #endif 769 770 #define __token(_tok, _num) _tok##_num 771 #define _token(_tok, _num) __token(_tok, _num) 772 #define __reserved__ _token(__reserved, __counter__) 773 774 struct mcx_cap_device { 775 uint8_t reserved0[16]; 776 777 uint8_t log_max_srq_sz; 778 uint8_t log_max_qp_sz; 779 uint8_t __reserved__[1]; 780 uint8_t log_max_qp; /* 5 bits */ 781 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 782 783 uint8_t __reserved__[1]; 784 uint8_t log_max_srq; /* 5 bits */ 785 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 786 uint8_t __reserved__[2]; 787 788 uint8_t __reserved__[1]; 789 uint8_t log_max_cq_sz; 790 uint8_t __reserved__[1]; 791 uint8_t log_max_cq; /* 5 bits */ 792 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 793 794 uint8_t log_max_eq_sz; 795 uint8_t log_max_mkey; /* 6 bits */ 796 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 797 uint8_t __reserved__[1]; 798 uint8_t log_max_eq; /* 4 bits */ 799 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 800 801 uint8_t max_indirection; 802 uint8_t log_max_mrw_sz; /* 7 bits */ 803 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 804 uint8_t teardown_log_max_msf_list_size; 805 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 806 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 807 0x3f 808 uint8_t log_max_klm_list_size; /* 6 bits */ 809 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 810 0x3f 811 812 uint8_t __reserved__[1]; 813 uint8_t log_max_ra_req_dc; /* 6 bits */ 814 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 815 uint8_t __reserved__[1]; 816 uint8_t log_max_ra_res_dc; /* 6 bits */ 817 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 818 0x3f 819 820 uint8_t __reserved__[1]; 821 uint8_t log_max_ra_req_qp; /* 6 bits */ 822 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 823 0x3f 824 uint8_t __reserved__[1]; 825 uint8_t log_max_ra_res_qp; /* 6 bits */ 826 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 827 0x3f 828 829 uint8_t flags1; 830 #define MCX_CAP_DEVICE_END_PAD 0x80 831 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 832 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 833 0x20 834 #define MCX_CAP_DEVICE_START_PAD 0x10 835 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 836 0x08 837 uint8_t __reserved__[1]; 838 uint16_t gid_table_size; 839 840 uint16_t flags2; 841 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 842 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 843 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 844 0x2000 845 #define MCX_CAP_DEVICE_DEBUG 0x1000 846 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 847 0x8000 848 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 849 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 850 uint16_t pkey_table_size; 851 852 uint8_t flags3; 853 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 854 0x80 855 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 856 0x40 857 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 858 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 859 #define MCX_CAP_DEVICE_ETS 0x04 860 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 861 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 862 0x01 863 uint8_t local_ca_ack_delay; /* 5 bits */ 864 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 865 0x1f 866 #define MCX_CAP_DEVICE_MCAM_REG 0x40 867 uint8_t port_type; 868 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 869 0x80 870 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 871 #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01 872 uint8_t num_ports; 873 874 uint8_t snapshot_log_max_msg; 875 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 876 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 877 uint8_t max_tc; /* 4 bits */ 878 #define MCX_CAP_DEVICE_MAX_TC 0x0f 879 uint8_t flags4; 880 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 881 #define MCX_CAP_DEVICE_DCBX 0x40 882 #define MCX_CAP_DEVICE_ROL_S 0x02 883 #define MCX_CAP_DEVICE_ROL_G 0x01 884 uint8_t wol; 885 #define MCX_CAP_DEVICE_WOL_S 0x40 886 #define MCX_CAP_DEVICE_WOL_G 0x20 887 #define MCX_CAP_DEVICE_WOL_A 0x10 888 #define MCX_CAP_DEVICE_WOL_B 0x08 889 #define MCX_CAP_DEVICE_WOL_M 0x04 890 #define MCX_CAP_DEVICE_WOL_U 0x02 891 #define MCX_CAP_DEVICE_WOL_P 0x01 892 893 uint16_t stat_rate_support; 894 uint8_t __reserved__[1]; 895 uint8_t cqe_version; /* 4 bits */ 896 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 897 898 uint32_t flags5; 899 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 900 0x80000000 901 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 902 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 903 0x10000000 904 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 905 0x08000000 906 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 907 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 908 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 909 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 910 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 911 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 912 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 913 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 914 #define MCX_CAP_DEVICE_SHO 0x00000100 915 #define MCX_CAP_DEVICE_TPH 0x00000080 916 #define MCX_CAP_DEVICE_RF 0x00000040 917 #define MCX_CAP_DEVICE_DCT 0x00000020 918 #define MCX_CAP_DEVICE_QOS 0x00000010 919 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 920 #define MCX_CAP_DEVICE_ROCE 0x00000004 921 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 922 923 uint32_t flags6; 924 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 925 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 926 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 927 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 928 0x10000000 929 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 930 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 931 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 932 #define MCX_CAP_DEVICE_PG 0x01000000 933 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 934 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 935 0x00400000 936 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 937 0x00200000 938 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 939 0x00100000 940 #define MCX_CAP_DEVICE_CD 0x00080000 941 #define MCX_CAP_DEVICE_ATM 0x00040000 942 #define MCX_CAP_DEVICE_APM 0x00020000 943 #define MCX_CAP_DEVICE_IMAICL 0x00010000 944 #define MCX_CAP_DEVICE_QKV 0x00000200 945 #define MCX_CAP_DEVICE_PKV 0x00000100 946 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 947 #define MCX_CAP_DEVICE_XRC 0x00000008 948 #define MCX_CAP_DEVICE_UD 0x00000004 949 #define MCX_CAP_DEVICE_UC 0x00000002 950 #define MCX_CAP_DEVICE_RC 0x00000001 951 952 uint8_t uar_flags; 953 #define MCX_CAP_DEVICE_UAR_4K 0x80 954 uint8_t uar_sz; /* 6 bits */ 955 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 956 uint8_t __reserved__[1]; 957 uint8_t log_pg_sz; 958 959 uint8_t flags7; 960 #define MCX_CAP_DEVICE_BF 0x80 961 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 962 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 963 0x20 964 uint8_t log_bf_reg_size; /* 5 bits */ 965 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 966 uint8_t __reserved__[2]; 967 968 uint16_t num_of_diagnostic_counters; 969 uint16_t max_wqe_sz_sq; 970 971 uint8_t __reserved__[2]; 972 uint16_t max_wqe_sz_rq; 973 974 uint8_t __reserved__[2]; 975 uint16_t max_wqe_sz_sq_dc; 976 977 uint32_t max_qp_mcg; /* 25 bits */ 978 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 979 980 uint8_t __reserved__[3]; 981 uint8_t log_max_mcq; 982 983 uint8_t log_max_transport_domain; /* 5 bits */ 984 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 985 0x1f 986 uint8_t log_max_pd; /* 5 bits */ 987 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 988 uint8_t __reserved__[1]; 989 uint8_t log_max_xrcd; /* 5 bits */ 990 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 991 992 uint8_t __reserved__[2]; 993 uint16_t max_flow_counter; 994 995 uint8_t log_max_rq; /* 5 bits */ 996 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 997 uint8_t log_max_sq; /* 5 bits */ 998 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 999 uint8_t log_max_tir; /* 5 bits */ 1000 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 1001 uint8_t log_max_tis; /* 5 bits */ 1002 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 1003 1004 uint8_t flags8; 1005 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 1006 0x80 1007 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 1008 uint8_t log_max_rqt; /* 5 bits */ 1009 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 1010 uint8_t log_max_rqt_size; /* 5 bits */ 1011 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 1012 uint8_t log_max_tis_per_sq; /* 5 bits */ 1013 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 1014 0x1f 1015 1016 uint8_t flags9; 1017 #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \ 1018 0x80 1019 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \ 1020 0x1f 1021 uint8_t log_min_stride_sz_rq; /* 5 bits */ 1022 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \ 1023 0x1f 1024 uint8_t log_max_stride_sz_sq; /* 5 bits */ 1025 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \ 1026 0x1f 1027 uint8_t log_min_stride_sz_sq; /* 5 bits */ 1028 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \ 1029 0x1f 1030 1031 uint8_t log_max_hairpin_queues; 1032 #define MXC_CAP_DEVICE_HAIRPIN 0x80 1033 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \ 1034 0x1f 1035 uint8_t log_min_hairpin_queues; 1036 #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \ 1037 0x1f 1038 uint8_t log_max_hairpin_num_packets; 1039 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \ 1040 0x1f 1041 uint8_t log_max_mq_sz; 1042 #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \ 1043 0x1f 1044 1045 uint8_t log_min_hairpin_wq_data_sz; 1046 #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \ 1047 0x80 1048 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \ 1049 0x40 1050 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \ 1051 0x20 1052 #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \ 1053 0x1f 1054 uint8_t log_max_vlan_list; 1055 #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \ 1056 0x80 1057 #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \ 1058 0x1f 1059 uint8_t log_max_current_mc_list; 1060 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \ 1061 0x1f 1062 uint8_t log_max_current_uc_list; 1063 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \ 1064 0x1f 1065 1066 uint8_t __reserved__[4]; 1067 1068 uint32_t create_qp_start_hint; /* 24 bits */ 1069 1070 uint8_t log_max_uctx; /* 5 bits */ 1071 #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f 1072 uint8_t log_max_umem; /* 5 bits */ 1073 #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f 1074 uint16_t max_num_eqs; 1075 1076 uint8_t log_max_l2_table; /* 5 bits */ 1077 #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f 1078 uint8_t __reserved__[1]; 1079 uint16_t log_uar_page_sz; 1080 1081 uint8_t __reserved__[8]; 1082 1083 uint32_t device_frequency_mhz; 1084 uint32_t device_frequency_khz; 1085 } __packed __aligned(8); 1086 1087 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 1088 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 1089 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 1090 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 1091 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 1092 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 1093 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98); 1094 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c); 1095 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 1096 1097 struct mcx_cmd_set_driver_version_in { 1098 uint16_t cmd_opcode; 1099 uint8_t cmd_reserved0[4]; 1100 uint16_t cmd_op_mod; 1101 uint8_t cmd_reserved1[8]; 1102 } __packed __aligned(4); 1103 1104 struct mcx_cmd_set_driver_version_out { 1105 uint8_t cmd_status; 1106 uint8_t cmd_reserved0[3]; 1107 uint32_t cmd_syndrome; 1108 uint8_t cmd_reserved1[8]; 1109 } __packed __aligned(4); 1110 1111 struct mcx_cmd_set_driver_version { 1112 uint8_t cmd_driver_version[64]; 1113 } __packed __aligned(8); 1114 1115 struct mcx_cmd_modify_nic_vport_context_in { 1116 uint16_t cmd_opcode; 1117 uint8_t cmd_reserved0[4]; 1118 uint16_t cmd_op_mod; 1119 uint8_t cmd_reserved1[4]; 1120 uint32_t cmd_field_select; 1121 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1122 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1123 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1124 } __packed __aligned(4); 1125 1126 struct mcx_cmd_modify_nic_vport_context_out { 1127 uint8_t cmd_status; 1128 uint8_t cmd_reserved0[3]; 1129 uint32_t cmd_syndrome; 1130 uint8_t cmd_reserved1[8]; 1131 } __packed __aligned(4); 1132 1133 struct mcx_cmd_query_nic_vport_context_in { 1134 uint16_t cmd_opcode; 1135 uint8_t cmd_reserved0[4]; 1136 uint16_t cmd_op_mod; 1137 uint8_t cmd_reserved1[4]; 1138 uint8_t cmd_allowed_list_type; 1139 uint8_t cmd_reserved2[3]; 1140 } __packed __aligned(4); 1141 1142 struct mcx_cmd_query_nic_vport_context_out { 1143 uint8_t cmd_status; 1144 uint8_t cmd_reserved0[3]; 1145 uint32_t cmd_syndrome; 1146 uint8_t cmd_reserved1[8]; 1147 } __packed __aligned(4); 1148 1149 struct mcx_nic_vport_ctx { 1150 uint32_t vp_min_wqe_inline_mode; 1151 uint8_t vp_reserved0[32]; 1152 uint32_t vp_mtu; 1153 uint8_t vp_reserved1[200]; 1154 uint16_t vp_flags; 1155 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1156 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1157 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1158 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1159 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1160 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1161 uint16_t vp_allowed_list_size; 1162 uint64_t vp_perm_addr; 1163 uint8_t vp_reserved2[4]; 1164 /* allowed list follows */ 1165 } __packed __aligned(4); 1166 1167 struct mcx_counter { 1168 uint64_t packets; 1169 uint64_t octets; 1170 } __packed __aligned(4); 1171 1172 struct mcx_nic_vport_counters { 1173 struct mcx_counter rx_err; 1174 struct mcx_counter tx_err; 1175 uint8_t reserved0[64]; /* 0x30 */ 1176 struct mcx_counter rx_bcast; 1177 struct mcx_counter tx_bcast; 1178 struct mcx_counter rx_ucast; 1179 struct mcx_counter tx_ucast; 1180 struct mcx_counter rx_mcast; 1181 struct mcx_counter tx_mcast; 1182 uint8_t reserved1[0x210 - 0xd0]; 1183 } __packed __aligned(4); 1184 1185 struct mcx_cmd_query_vport_counters_in { 1186 uint16_t cmd_opcode; 1187 uint8_t cmd_reserved0[4]; 1188 uint16_t cmd_op_mod; 1189 uint8_t cmd_reserved1[8]; 1190 } __packed __aligned(4); 1191 1192 struct mcx_cmd_query_vport_counters_mb_in { 1193 uint8_t cmd_reserved0[8]; 1194 uint8_t cmd_clear; 1195 uint8_t cmd_reserved1[7]; 1196 } __packed __aligned(4); 1197 1198 struct mcx_cmd_query_vport_counters_out { 1199 uint8_t cmd_status; 1200 uint8_t cmd_reserved0[3]; 1201 uint32_t cmd_syndrome; 1202 uint8_t cmd_reserved1[8]; 1203 } __packed __aligned(4); 1204 1205 struct mcx_cmd_query_flow_counter_in { 1206 uint16_t cmd_opcode; 1207 uint8_t cmd_reserved0[4]; 1208 uint16_t cmd_op_mod; 1209 uint8_t cmd_reserved1[8]; 1210 } __packed __aligned(4); 1211 1212 struct mcx_cmd_query_flow_counter_mb_in { 1213 uint8_t cmd_reserved0[8]; 1214 uint8_t cmd_clear; 1215 uint8_t cmd_reserved1[5]; 1216 uint16_t cmd_flow_counter_id; 1217 } __packed __aligned(4); 1218 1219 struct mcx_cmd_query_flow_counter_out { 1220 uint8_t cmd_status; 1221 uint8_t cmd_reserved0[3]; 1222 uint32_t cmd_syndrome; 1223 uint8_t cmd_reserved1[8]; 1224 } __packed __aligned(4); 1225 1226 struct mcx_cmd_alloc_uar_in { 1227 uint16_t cmd_opcode; 1228 uint8_t cmd_reserved0[4]; 1229 uint16_t cmd_op_mod; 1230 uint8_t cmd_reserved1[8]; 1231 } __packed __aligned(4); 1232 1233 struct mcx_cmd_alloc_uar_out { 1234 uint8_t cmd_status; 1235 uint8_t cmd_reserved0[3]; 1236 uint32_t cmd_syndrome; 1237 uint32_t cmd_uar; 1238 uint8_t cmd_reserved1[4]; 1239 } __packed __aligned(4); 1240 1241 struct mcx_cmd_query_special_ctx_in { 1242 uint16_t cmd_opcode; 1243 uint8_t cmd_reserved0[4]; 1244 uint16_t cmd_op_mod; 1245 uint8_t cmd_reserved1[8]; 1246 } __packed __aligned(4); 1247 1248 struct mcx_cmd_query_special_ctx_out { 1249 uint8_t cmd_status; 1250 uint8_t cmd_reserved0[3]; 1251 uint32_t cmd_syndrome; 1252 uint8_t cmd_reserved1[4]; 1253 uint32_t cmd_resd_lkey; 1254 } __packed __aligned(4); 1255 1256 struct mcx_eq_ctx { 1257 uint32_t eq_status; 1258 #define MCX_EQ_CTX_STATE_SHIFT 8 1259 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT) 1260 #define MCX_EQ_CTX_STATE_ARMED 0x9 1261 #define MCX_EQ_CTX_STATE_FIRED 0xa 1262 #define MCX_EQ_CTX_OI_SHIFT 17 1263 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1264 #define MCX_EQ_CTX_EC_SHIFT 18 1265 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1266 #define MCX_EQ_CTX_STATUS_SHIFT 28 1267 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1268 #define MCX_EQ_CTX_STATUS_OK 0x0 1269 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa 1270 uint32_t eq_reserved1; 1271 uint32_t eq_page_offset; 1272 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1273 uint32_t eq_uar_size; 1274 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1275 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1276 uint32_t eq_reserved2; 1277 uint8_t eq_reserved3[3]; 1278 uint8_t eq_intr; 1279 uint32_t eq_log_page_size; 1280 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1281 uint32_t eq_reserved4[3]; 1282 uint32_t eq_consumer_counter; 1283 uint32_t eq_producer_counter; 1284 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1285 uint32_t eq_reserved5[4]; 1286 } __packed __aligned(4); 1287 1288 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1289 1290 struct mcx_cmd_create_eq_in { 1291 uint16_t cmd_opcode; 1292 uint8_t cmd_reserved0[4]; 1293 uint16_t cmd_op_mod; 1294 uint8_t cmd_reserved1[8]; 1295 } __packed __aligned(4); 1296 1297 struct mcx_cmd_create_eq_mb_in { 1298 struct mcx_eq_ctx cmd_eq_ctx; 1299 uint8_t cmd_reserved0[8]; 1300 uint64_t cmd_event_bitmask; 1301 #define MCX_EVENT_TYPE_COMPLETION 0x00 1302 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1303 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1304 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1305 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1306 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1307 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1308 uint8_t cmd_reserved1[176]; 1309 } __packed __aligned(4); 1310 1311 struct mcx_cmd_create_eq_out { 1312 uint8_t cmd_status; 1313 uint8_t cmd_reserved0[3]; 1314 uint32_t cmd_syndrome; 1315 uint32_t cmd_eqn; 1316 uint8_t cmd_reserved1[4]; 1317 } __packed __aligned(4); 1318 1319 struct mcx_cmd_query_eq_in { 1320 uint16_t cmd_opcode; 1321 uint8_t cmd_reserved0[4]; 1322 uint16_t cmd_op_mod; 1323 uint32_t cmd_eqn; 1324 uint8_t cmd_reserved1[4]; 1325 } __packed __aligned(4); 1326 1327 struct mcx_cmd_query_eq_out { 1328 uint8_t cmd_status; 1329 uint8_t cmd_reserved0[3]; 1330 uint32_t cmd_syndrome; 1331 uint8_t cmd_reserved1[8]; 1332 } __packed __aligned(4); 1333 1334 struct mcx_eq_entry { 1335 uint8_t eq_reserved1; 1336 uint8_t eq_event_type; 1337 uint8_t eq_reserved2; 1338 uint8_t eq_event_sub_type; 1339 1340 uint8_t eq_reserved3[28]; 1341 uint32_t eq_event_data[7]; 1342 uint8_t eq_reserved4[2]; 1343 uint8_t eq_signature; 1344 uint8_t eq_owner; 1345 #define MCX_EQ_ENTRY_OWNER_INIT 1 1346 } __packed __aligned(4); 1347 1348 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1349 1350 struct mcx_cmd_alloc_pd_in { 1351 uint16_t cmd_opcode; 1352 uint8_t cmd_reserved0[4]; 1353 uint16_t cmd_op_mod; 1354 uint8_t cmd_reserved1[8]; 1355 } __packed __aligned(4); 1356 1357 struct mcx_cmd_alloc_pd_out { 1358 uint8_t cmd_status; 1359 uint8_t cmd_reserved0[3]; 1360 uint32_t cmd_syndrome; 1361 uint32_t cmd_pd; 1362 uint8_t cmd_reserved1[4]; 1363 } __packed __aligned(4); 1364 1365 struct mcx_cmd_alloc_td_in { 1366 uint16_t cmd_opcode; 1367 uint8_t cmd_reserved0[4]; 1368 uint16_t cmd_op_mod; 1369 uint8_t cmd_reserved1[8]; 1370 } __packed __aligned(4); 1371 1372 struct mcx_cmd_alloc_td_out { 1373 uint8_t cmd_status; 1374 uint8_t cmd_reserved0[3]; 1375 uint32_t cmd_syndrome; 1376 uint32_t cmd_tdomain; 1377 uint8_t cmd_reserved1[4]; 1378 } __packed __aligned(4); 1379 1380 struct mcx_cmd_create_tir_in { 1381 uint16_t cmd_opcode; 1382 uint8_t cmd_reserved0[4]; 1383 uint16_t cmd_op_mod; 1384 uint8_t cmd_reserved1[8]; 1385 } __packed __aligned(4); 1386 1387 struct mcx_cmd_create_tir_mb_in { 1388 uint8_t cmd_reserved0[20]; 1389 uint32_t cmd_disp_type; 1390 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1391 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1392 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1393 uint8_t cmd_reserved1[8]; 1394 uint32_t cmd_lro; 1395 uint8_t cmd_reserved2[8]; 1396 uint32_t cmd_inline_rqn; 1397 uint32_t cmd_indir_table; 1398 uint32_t cmd_tdomain; 1399 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1400 #define MCX_TIR_CTX_HASH_SHIFT 28 1401 uint8_t cmd_rx_hash_key[40]; 1402 uint32_t cmd_rx_hash_sel_outer; 1403 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1404 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1405 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1406 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1407 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1408 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1409 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1410 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1411 uint32_t cmd_rx_hash_sel_inner; 1412 uint8_t cmd_reserved3[152]; 1413 } __packed __aligned(4); 1414 1415 struct mcx_cmd_create_tir_out { 1416 uint8_t cmd_status; 1417 uint8_t cmd_reserved0[3]; 1418 uint32_t cmd_syndrome; 1419 uint32_t cmd_tirn; 1420 uint8_t cmd_reserved1[4]; 1421 } __packed __aligned(4); 1422 1423 struct mcx_cmd_destroy_tir_in { 1424 uint16_t cmd_opcode; 1425 uint8_t cmd_reserved0[4]; 1426 uint16_t cmd_op_mod; 1427 uint32_t cmd_tirn; 1428 uint8_t cmd_reserved1[4]; 1429 } __packed __aligned(4); 1430 1431 struct mcx_cmd_destroy_tir_out { 1432 uint8_t cmd_status; 1433 uint8_t cmd_reserved0[3]; 1434 uint32_t cmd_syndrome; 1435 uint8_t cmd_reserved1[8]; 1436 } __packed __aligned(4); 1437 1438 struct mcx_cmd_create_tis_in { 1439 uint16_t cmd_opcode; 1440 uint8_t cmd_reserved0[4]; 1441 uint16_t cmd_op_mod; 1442 uint8_t cmd_reserved1[8]; 1443 } __packed __aligned(4); 1444 1445 struct mcx_cmd_create_tis_mb_in { 1446 uint8_t cmd_reserved[16]; 1447 uint32_t cmd_prio; 1448 uint8_t cmd_reserved1[32]; 1449 uint32_t cmd_tdomain; 1450 uint8_t cmd_reserved2[120]; 1451 } __packed __aligned(4); 1452 1453 struct mcx_cmd_create_tis_out { 1454 uint8_t cmd_status; 1455 uint8_t cmd_reserved0[3]; 1456 uint32_t cmd_syndrome; 1457 uint32_t cmd_tisn; 1458 uint8_t cmd_reserved1[4]; 1459 } __packed __aligned(4); 1460 1461 struct mcx_cmd_destroy_tis_in { 1462 uint16_t cmd_opcode; 1463 uint8_t cmd_reserved0[4]; 1464 uint16_t cmd_op_mod; 1465 uint32_t cmd_tisn; 1466 uint8_t cmd_reserved1[4]; 1467 } __packed __aligned(4); 1468 1469 struct mcx_cmd_destroy_tis_out { 1470 uint8_t cmd_status; 1471 uint8_t cmd_reserved0[3]; 1472 uint32_t cmd_syndrome; 1473 uint8_t cmd_reserved1[8]; 1474 } __packed __aligned(4); 1475 1476 struct mcx_cmd_create_rqt_in { 1477 uint16_t cmd_opcode; 1478 uint8_t cmd_reserved0[4]; 1479 uint16_t cmd_op_mod; 1480 uint8_t cmd_reserved1[8]; 1481 } __packed __aligned(4); 1482 1483 struct mcx_rqt_ctx { 1484 uint8_t cmd_reserved0[20]; 1485 uint16_t cmd_reserved1; 1486 uint16_t cmd_rqt_max_size; 1487 uint16_t cmd_reserved2; 1488 uint16_t cmd_rqt_actual_size; 1489 uint8_t cmd_reserved3[212]; 1490 } __packed __aligned(4); 1491 1492 struct mcx_cmd_create_rqt_mb_in { 1493 uint8_t cmd_reserved0[16]; 1494 struct mcx_rqt_ctx cmd_rqt; 1495 } __packed __aligned(4); 1496 1497 struct mcx_cmd_create_rqt_out { 1498 uint8_t cmd_status; 1499 uint8_t cmd_reserved0[3]; 1500 uint32_t cmd_syndrome; 1501 uint32_t cmd_rqtn; 1502 uint8_t cmd_reserved1[4]; 1503 } __packed __aligned(4); 1504 1505 struct mcx_cmd_destroy_rqt_in { 1506 uint16_t cmd_opcode; 1507 uint8_t cmd_reserved0[4]; 1508 uint16_t cmd_op_mod; 1509 uint32_t cmd_rqtn; 1510 uint8_t cmd_reserved1[4]; 1511 } __packed __aligned(4); 1512 1513 struct mcx_cmd_destroy_rqt_out { 1514 uint8_t cmd_status; 1515 uint8_t cmd_reserved0[3]; 1516 uint32_t cmd_syndrome; 1517 uint8_t cmd_reserved1[8]; 1518 } __packed __aligned(4); 1519 1520 struct mcx_cq_ctx { 1521 uint32_t cq_status; 1522 #define MCX_CQ_CTX_STATUS_SHIFT 28 1523 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT) 1524 #define MCX_CQ_CTX_STATUS_OK 0x0 1525 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9 1526 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa 1527 #define MCX_CQ_CTX_STATE_SHIFT 8 1528 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT) 1529 #define MCX_CQ_CTX_STATE_SOLICITED 0x6 1530 #define MCX_CQ_CTX_STATE_ARMED 0x9 1531 #define MCX_CQ_CTX_STATE_FIRED 0xa 1532 uint32_t cq_reserved1; 1533 uint32_t cq_page_offset; 1534 uint32_t cq_uar_size; 1535 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1536 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1537 uint32_t cq_period_max_count; 1538 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1539 uint32_t cq_eqn; 1540 uint32_t cq_log_page_size; 1541 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1542 uint32_t cq_reserved2; 1543 uint32_t cq_last_notified; 1544 uint32_t cq_last_solicit; 1545 uint32_t cq_consumer_counter; 1546 uint32_t cq_producer_counter; 1547 uint8_t cq_reserved3[8]; 1548 uint64_t cq_doorbell; 1549 } __packed __aligned(4); 1550 1551 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1552 1553 struct mcx_cmd_create_cq_in { 1554 uint16_t cmd_opcode; 1555 uint8_t cmd_reserved0[4]; 1556 uint16_t cmd_op_mod; 1557 uint8_t cmd_reserved1[8]; 1558 } __packed __aligned(4); 1559 1560 struct mcx_cmd_create_cq_mb_in { 1561 struct mcx_cq_ctx cmd_cq_ctx; 1562 uint8_t cmd_reserved1[192]; 1563 } __packed __aligned(4); 1564 1565 struct mcx_cmd_create_cq_out { 1566 uint8_t cmd_status; 1567 uint8_t cmd_reserved0[3]; 1568 uint32_t cmd_syndrome; 1569 uint32_t cmd_cqn; 1570 uint8_t cmd_reserved1[4]; 1571 } __packed __aligned(4); 1572 1573 struct mcx_cmd_destroy_cq_in { 1574 uint16_t cmd_opcode; 1575 uint8_t cmd_reserved0[4]; 1576 uint16_t cmd_op_mod; 1577 uint32_t cmd_cqn; 1578 uint8_t cmd_reserved1[4]; 1579 } __packed __aligned(4); 1580 1581 struct mcx_cmd_destroy_cq_out { 1582 uint8_t cmd_status; 1583 uint8_t cmd_reserved0[3]; 1584 uint32_t cmd_syndrome; 1585 uint8_t cmd_reserved1[8]; 1586 } __packed __aligned(4); 1587 1588 struct mcx_cmd_query_cq_in { 1589 uint16_t cmd_opcode; 1590 uint8_t cmd_reserved0[4]; 1591 uint16_t cmd_op_mod; 1592 uint32_t cmd_cqn; 1593 uint8_t cmd_reserved1[4]; 1594 } __packed __aligned(4); 1595 1596 struct mcx_cmd_query_cq_out { 1597 uint8_t cmd_status; 1598 uint8_t cmd_reserved0[3]; 1599 uint32_t cmd_syndrome; 1600 uint8_t cmd_reserved1[8]; 1601 } __packed __aligned(4); 1602 1603 struct mcx_cq_entry { 1604 uint32_t __reserved__; 1605 uint32_t cq_lro; 1606 uint32_t cq_lro_ack_seq_num; 1607 uint32_t cq_rx_hash; 1608 uint8_t cq_rx_hash_type; 1609 uint8_t cq_ml_path; 1610 uint16_t __reserved__; 1611 uint32_t cq_checksum; 1612 uint32_t __reserved__; 1613 uint32_t cq_flags; 1614 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1615 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1616 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1617 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1618 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1619 1620 uint32_t cq_lro_srqn; 1621 uint32_t __reserved__[2]; 1622 uint32_t cq_byte_cnt; 1623 uint64_t cq_timestamp; 1624 uint8_t cq_rx_drops; 1625 uint8_t cq_flow_tag[3]; 1626 uint16_t cq_wqe_count; 1627 uint8_t cq_signature; 1628 uint8_t cq_opcode_owner; 1629 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1630 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1631 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1632 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1633 1634 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1635 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1636 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1637 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1638 1639 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1640 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1641 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1642 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1643 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1644 1645 } __packed __aligned(4); 1646 1647 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1648 1649 struct mcx_cq_doorbell { 1650 uint32_t db_update_ci; 1651 uint32_t db_arm_ci; 1652 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1653 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1654 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1655 } __packed __aligned(8); 1656 1657 struct mcx_wq_ctx { 1658 uint8_t wq_type; 1659 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1660 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1661 uint8_t wq_reserved0[5]; 1662 uint16_t wq_lwm; 1663 uint32_t wq_pd; 1664 uint32_t wq_uar_page; 1665 uint64_t wq_doorbell; 1666 uint32_t wq_hw_counter; 1667 uint32_t wq_sw_counter; 1668 uint16_t wq_log_stride; 1669 uint8_t wq_log_page_sz; 1670 uint8_t wq_log_size; 1671 uint8_t wq_reserved1[156]; 1672 } __packed __aligned(4); 1673 1674 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1675 1676 struct mcx_sq_ctx { 1677 uint32_t sq_flags; 1678 #define MCX_SQ_CTX_RLKEY (1 << 31) 1679 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1680 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1681 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1682 #define MCX_SQ_CTX_STATE_SHIFT 20 1683 #define MCX_SQ_CTX_STATE_MASK (0xf << 20) 1684 #define MCX_SQ_CTX_STATE_RST 0 1685 #define MCX_SQ_CTX_STATE_RDY 1 1686 #define MCX_SQ_CTX_STATE_ERR 3 1687 uint32_t sq_user_index; 1688 uint32_t sq_cqn; 1689 uint32_t sq_reserved1[5]; 1690 uint32_t sq_tis_lst_sz; 1691 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1692 uint32_t sq_reserved2[2]; 1693 uint32_t sq_tis_num; 1694 struct mcx_wq_ctx sq_wq; 1695 } __packed __aligned(4); 1696 1697 struct mcx_sq_entry_seg { 1698 uint32_t sqs_byte_count; 1699 uint32_t sqs_lkey; 1700 uint64_t sqs_addr; 1701 } __packed __aligned(4); 1702 1703 struct mcx_sq_entry { 1704 /* control segment */ 1705 uint32_t sqe_opcode_index; 1706 #define MCX_SQE_WQE_INDEX_SHIFT 8 1707 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1708 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1709 uint32_t sqe_ds_sq_num; 1710 #define MCX_SQE_SQ_NUM_SHIFT 8 1711 uint32_t sqe_signature; 1712 #define MCX_SQE_SIGNATURE_SHIFT 24 1713 #define MCX_SQE_SOLICITED_EVENT 0x02 1714 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1715 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1716 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1717 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1718 #define MCX_SQE_FM_NO_FENCE 0x00 1719 #define MCX_SQE_FM_SMALL_FENCE 0x40 1720 uint32_t sqe_mkey; 1721 1722 /* ethernet segment */ 1723 uint32_t sqe_reserved1; 1724 uint32_t sqe_mss_csum; 1725 #define MCX_SQE_L4_CSUM (1 << 31) 1726 #define MCX_SQE_L3_CSUM (1 << 30) 1727 uint32_t sqe_reserved2; 1728 uint16_t sqe_inline_header_size; 1729 uint16_t sqe_inline_headers[9]; 1730 1731 /* data segment */ 1732 struct mcx_sq_entry_seg sqe_segs[1]; 1733 } __packed __aligned(64); 1734 1735 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1736 1737 struct mcx_cmd_create_sq_in { 1738 uint16_t cmd_opcode; 1739 uint8_t cmd_reserved0[4]; 1740 uint16_t cmd_op_mod; 1741 uint8_t cmd_reserved1[8]; 1742 } __packed __aligned(4); 1743 1744 struct mcx_cmd_create_sq_out { 1745 uint8_t cmd_status; 1746 uint8_t cmd_reserved0[3]; 1747 uint32_t cmd_syndrome; 1748 uint32_t cmd_sqn; 1749 uint8_t cmd_reserved1[4]; 1750 } __packed __aligned(4); 1751 1752 struct mcx_cmd_modify_sq_in { 1753 uint16_t cmd_opcode; 1754 uint8_t cmd_reserved0[4]; 1755 uint16_t cmd_op_mod; 1756 uint32_t cmd_sq_state; 1757 uint8_t cmd_reserved1[4]; 1758 } __packed __aligned(4); 1759 1760 struct mcx_cmd_modify_sq_mb_in { 1761 uint32_t cmd_modify_hi; 1762 uint32_t cmd_modify_lo; 1763 uint8_t cmd_reserved0[8]; 1764 struct mcx_sq_ctx cmd_sq_ctx; 1765 } __packed __aligned(4); 1766 1767 struct mcx_cmd_modify_sq_out { 1768 uint8_t cmd_status; 1769 uint8_t cmd_reserved0[3]; 1770 uint32_t cmd_syndrome; 1771 uint8_t cmd_reserved1[8]; 1772 } __packed __aligned(4); 1773 1774 struct mcx_cmd_destroy_sq_in { 1775 uint16_t cmd_opcode; 1776 uint8_t cmd_reserved0[4]; 1777 uint16_t cmd_op_mod; 1778 uint32_t cmd_sqn; 1779 uint8_t cmd_reserved1[4]; 1780 } __packed __aligned(4); 1781 1782 struct mcx_cmd_destroy_sq_out { 1783 uint8_t cmd_status; 1784 uint8_t cmd_reserved0[3]; 1785 uint32_t cmd_syndrome; 1786 uint8_t cmd_reserved1[8]; 1787 } __packed __aligned(4); 1788 1789 1790 struct mcx_rq_ctx { 1791 uint32_t rq_flags; 1792 #define MCX_RQ_CTX_RLKEY (1 << 31) 1793 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1794 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1795 #define MCX_RQ_CTX_STATE_SHIFT 20 1796 #define MCX_RQ_CTX_STATE_MASK (0xf << 20) 1797 #define MCX_RQ_CTX_STATE_RST 0 1798 #define MCX_RQ_CTX_STATE_RDY 1 1799 #define MCX_RQ_CTX_STATE_ERR 3 1800 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1801 uint32_t rq_user_index; 1802 uint32_t rq_cqn; 1803 uint32_t rq_reserved1; 1804 uint32_t rq_rmpn; 1805 uint32_t rq_reserved2[7]; 1806 struct mcx_wq_ctx rq_wq; 1807 } __packed __aligned(4); 1808 1809 struct mcx_rq_entry { 1810 uint32_t rqe_byte_count; 1811 uint32_t rqe_lkey; 1812 uint64_t rqe_addr; 1813 } __packed __aligned(16); 1814 1815 struct mcx_cmd_create_rq_in { 1816 uint16_t cmd_opcode; 1817 uint8_t cmd_reserved0[4]; 1818 uint16_t cmd_op_mod; 1819 uint8_t cmd_reserved1[8]; 1820 } __packed __aligned(4); 1821 1822 struct mcx_cmd_create_rq_out { 1823 uint8_t cmd_status; 1824 uint8_t cmd_reserved0[3]; 1825 uint32_t cmd_syndrome; 1826 uint32_t cmd_rqn; 1827 uint8_t cmd_reserved1[4]; 1828 } __packed __aligned(4); 1829 1830 struct mcx_cmd_modify_rq_in { 1831 uint16_t cmd_opcode; 1832 uint8_t cmd_reserved0[4]; 1833 uint16_t cmd_op_mod; 1834 uint32_t cmd_rq_state; 1835 uint8_t cmd_reserved1[4]; 1836 } __packed __aligned(4); 1837 1838 struct mcx_cmd_modify_rq_mb_in { 1839 uint32_t cmd_modify_hi; 1840 uint32_t cmd_modify_lo; 1841 uint8_t cmd_reserved0[8]; 1842 struct mcx_rq_ctx cmd_rq_ctx; 1843 } __packed __aligned(4); 1844 1845 struct mcx_cmd_modify_rq_out { 1846 uint8_t cmd_status; 1847 uint8_t cmd_reserved0[3]; 1848 uint32_t cmd_syndrome; 1849 uint8_t cmd_reserved1[8]; 1850 } __packed __aligned(4); 1851 1852 struct mcx_cmd_destroy_rq_in { 1853 uint16_t cmd_opcode; 1854 uint8_t cmd_reserved0[4]; 1855 uint16_t cmd_op_mod; 1856 uint32_t cmd_rqn; 1857 uint8_t cmd_reserved1[4]; 1858 } __packed __aligned(4); 1859 1860 struct mcx_cmd_destroy_rq_out { 1861 uint8_t cmd_status; 1862 uint8_t cmd_reserved0[3]; 1863 uint32_t cmd_syndrome; 1864 uint8_t cmd_reserved1[8]; 1865 } __packed __aligned(4); 1866 1867 struct mcx_cmd_create_flow_table_in { 1868 uint16_t cmd_opcode; 1869 uint8_t cmd_reserved0[4]; 1870 uint16_t cmd_op_mod; 1871 uint8_t cmd_reserved1[8]; 1872 } __packed __aligned(4); 1873 1874 struct mcx_flow_table_ctx { 1875 uint8_t ft_miss_action; 1876 uint8_t ft_level; 1877 uint8_t ft_reserved0; 1878 uint8_t ft_log_size; 1879 uint32_t ft_table_miss_id; 1880 uint8_t ft_reserved1[28]; 1881 } __packed __aligned(4); 1882 1883 struct mcx_cmd_create_flow_table_mb_in { 1884 uint8_t cmd_table_type; 1885 uint8_t cmd_reserved0[7]; 1886 struct mcx_flow_table_ctx cmd_ctx; 1887 } __packed __aligned(4); 1888 1889 struct mcx_cmd_create_flow_table_out { 1890 uint8_t cmd_status; 1891 uint8_t cmd_reserved0[3]; 1892 uint32_t cmd_syndrome; 1893 uint32_t cmd_table_id; 1894 uint8_t cmd_reserved1[4]; 1895 } __packed __aligned(4); 1896 1897 struct mcx_cmd_destroy_flow_table_in { 1898 uint16_t cmd_opcode; 1899 uint8_t cmd_reserved0[4]; 1900 uint16_t cmd_op_mod; 1901 uint8_t cmd_reserved1[8]; 1902 } __packed __aligned(4); 1903 1904 struct mcx_cmd_destroy_flow_table_mb_in { 1905 uint8_t cmd_table_type; 1906 uint8_t cmd_reserved0[3]; 1907 uint32_t cmd_table_id; 1908 uint8_t cmd_reserved1[40]; 1909 } __packed __aligned(4); 1910 1911 struct mcx_cmd_destroy_flow_table_out { 1912 uint8_t cmd_status; 1913 uint8_t cmd_reserved0[3]; 1914 uint32_t cmd_syndrome; 1915 uint8_t cmd_reserved1[8]; 1916 } __packed __aligned(4); 1917 1918 struct mcx_cmd_set_flow_table_root_in { 1919 uint16_t cmd_opcode; 1920 uint8_t cmd_reserved0[4]; 1921 uint16_t cmd_op_mod; 1922 uint8_t cmd_reserved1[8]; 1923 } __packed __aligned(4); 1924 1925 struct mcx_cmd_set_flow_table_root_mb_in { 1926 uint8_t cmd_table_type; 1927 uint8_t cmd_reserved0[3]; 1928 uint32_t cmd_table_id; 1929 uint8_t cmd_reserved1[56]; 1930 } __packed __aligned(4); 1931 1932 struct mcx_cmd_set_flow_table_root_out { 1933 uint8_t cmd_status; 1934 uint8_t cmd_reserved0[3]; 1935 uint32_t cmd_syndrome; 1936 uint8_t cmd_reserved1[8]; 1937 } __packed __aligned(4); 1938 1939 struct mcx_flow_match { 1940 /* outer headers */ 1941 uint8_t mc_src_mac[6]; 1942 uint16_t mc_ethertype; 1943 uint8_t mc_dest_mac[6]; 1944 uint16_t mc_first_vlan; 1945 uint8_t mc_ip_proto; 1946 uint8_t mc_ip_dscp_ecn; 1947 uint8_t mc_vlan_flags; 1948 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1949 uint8_t mc_tcp_flags; 1950 uint16_t mc_tcp_sport; 1951 uint16_t mc_tcp_dport; 1952 uint32_t mc_reserved0; 1953 uint16_t mc_udp_sport; 1954 uint16_t mc_udp_dport; 1955 uint8_t mc_src_ip[16]; 1956 uint8_t mc_dest_ip[16]; 1957 1958 /* misc parameters */ 1959 uint8_t mc_reserved1[8]; 1960 uint16_t mc_second_vlan; 1961 uint8_t mc_reserved2[2]; 1962 uint8_t mc_second_vlan_flags; 1963 uint8_t mc_reserved3[15]; 1964 uint32_t mc_outer_ipv6_flow_label; 1965 uint8_t mc_reserved4[32]; 1966 1967 uint8_t mc_reserved[384]; 1968 } __packed __aligned(4); 1969 1970 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1971 1972 struct mcx_cmd_create_flow_group_in { 1973 uint16_t cmd_opcode; 1974 uint8_t cmd_reserved0[4]; 1975 uint16_t cmd_op_mod; 1976 uint8_t cmd_reserved1[8]; 1977 } __packed __aligned(4); 1978 1979 struct mcx_cmd_create_flow_group_mb_in { 1980 uint8_t cmd_table_type; 1981 uint8_t cmd_reserved0[3]; 1982 uint32_t cmd_table_id; 1983 uint8_t cmd_reserved1[4]; 1984 uint32_t cmd_start_flow_index; 1985 uint8_t cmd_reserved2[4]; 1986 uint32_t cmd_end_flow_index; 1987 uint8_t cmd_reserved3[23]; 1988 uint8_t cmd_match_criteria_enable; 1989 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1990 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1991 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1992 struct mcx_flow_match cmd_match_criteria; 1993 uint8_t cmd_reserved4[448]; 1994 } __packed __aligned(4); 1995 1996 struct mcx_cmd_create_flow_group_out { 1997 uint8_t cmd_status; 1998 uint8_t cmd_reserved0[3]; 1999 uint32_t cmd_syndrome; 2000 uint32_t cmd_group_id; 2001 uint8_t cmd_reserved1[4]; 2002 } __packed __aligned(4); 2003 2004 struct mcx_flow_ctx { 2005 uint8_t fc_reserved0[4]; 2006 uint32_t fc_group_id; 2007 uint32_t fc_flow_tag; 2008 uint32_t fc_action; 2009 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 2010 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 2011 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 2012 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 2013 uint32_t fc_dest_list_size; 2014 uint32_t fc_counter_list_size; 2015 uint8_t fc_reserved1[40]; 2016 struct mcx_flow_match fc_match_value; 2017 uint8_t fc_reserved2[192]; 2018 } __packed __aligned(4); 2019 2020 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 2021 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 2022 2023 struct mcx_cmd_destroy_flow_group_in { 2024 uint16_t cmd_opcode; 2025 uint8_t cmd_reserved0[4]; 2026 uint16_t cmd_op_mod; 2027 uint8_t cmd_reserved1[8]; 2028 } __packed __aligned(4); 2029 2030 struct mcx_cmd_destroy_flow_group_mb_in { 2031 uint8_t cmd_table_type; 2032 uint8_t cmd_reserved0[3]; 2033 uint32_t cmd_table_id; 2034 uint32_t cmd_group_id; 2035 uint8_t cmd_reserved1[36]; 2036 } __packed __aligned(4); 2037 2038 struct mcx_cmd_destroy_flow_group_out { 2039 uint8_t cmd_status; 2040 uint8_t cmd_reserved0[3]; 2041 uint32_t cmd_syndrome; 2042 uint8_t cmd_reserved1[8]; 2043 } __packed __aligned(4); 2044 2045 struct mcx_cmd_set_flow_table_entry_in { 2046 uint16_t cmd_opcode; 2047 uint8_t cmd_reserved0[4]; 2048 uint16_t cmd_op_mod; 2049 uint8_t cmd_reserved1[8]; 2050 } __packed __aligned(4); 2051 2052 struct mcx_cmd_set_flow_table_entry_mb_in { 2053 uint8_t cmd_table_type; 2054 uint8_t cmd_reserved0[3]; 2055 uint32_t cmd_table_id; 2056 uint32_t cmd_modify_enable_mask; 2057 uint8_t cmd_reserved1[4]; 2058 uint32_t cmd_flow_index; 2059 uint8_t cmd_reserved2[28]; 2060 struct mcx_flow_ctx cmd_flow_ctx; 2061 } __packed __aligned(4); 2062 2063 struct mcx_cmd_set_flow_table_entry_out { 2064 uint8_t cmd_status; 2065 uint8_t cmd_reserved0[3]; 2066 uint32_t cmd_syndrome; 2067 uint8_t cmd_reserved1[8]; 2068 } __packed __aligned(4); 2069 2070 struct mcx_cmd_query_flow_table_entry_in { 2071 uint16_t cmd_opcode; 2072 uint8_t cmd_reserved0[4]; 2073 uint16_t cmd_op_mod; 2074 uint8_t cmd_reserved1[8]; 2075 } __packed __aligned(4); 2076 2077 struct mcx_cmd_query_flow_table_entry_mb_in { 2078 uint8_t cmd_table_type; 2079 uint8_t cmd_reserved0[3]; 2080 uint32_t cmd_table_id; 2081 uint8_t cmd_reserved1[8]; 2082 uint32_t cmd_flow_index; 2083 uint8_t cmd_reserved2[28]; 2084 } __packed __aligned(4); 2085 2086 struct mcx_cmd_query_flow_table_entry_out { 2087 uint8_t cmd_status; 2088 uint8_t cmd_reserved0[3]; 2089 uint32_t cmd_syndrome; 2090 uint8_t cmd_reserved1[8]; 2091 } __packed __aligned(4); 2092 2093 struct mcx_cmd_query_flow_table_entry_mb_out { 2094 uint8_t cmd_reserved0[48]; 2095 struct mcx_flow_ctx cmd_flow_ctx; 2096 } __packed __aligned(4); 2097 2098 struct mcx_cmd_delete_flow_table_entry_in { 2099 uint16_t cmd_opcode; 2100 uint8_t cmd_reserved0[4]; 2101 uint16_t cmd_op_mod; 2102 uint8_t cmd_reserved1[8]; 2103 } __packed __aligned(4); 2104 2105 struct mcx_cmd_delete_flow_table_entry_mb_in { 2106 uint8_t cmd_table_type; 2107 uint8_t cmd_reserved0[3]; 2108 uint32_t cmd_table_id; 2109 uint8_t cmd_reserved1[8]; 2110 uint32_t cmd_flow_index; 2111 uint8_t cmd_reserved2[28]; 2112 } __packed __aligned(4); 2113 2114 struct mcx_cmd_delete_flow_table_entry_out { 2115 uint8_t cmd_status; 2116 uint8_t cmd_reserved0[3]; 2117 uint32_t cmd_syndrome; 2118 uint8_t cmd_reserved1[8]; 2119 } __packed __aligned(4); 2120 2121 struct mcx_cmd_query_flow_group_in { 2122 uint16_t cmd_opcode; 2123 uint8_t cmd_reserved0[4]; 2124 uint16_t cmd_op_mod; 2125 uint8_t cmd_reserved1[8]; 2126 } __packed __aligned(4); 2127 2128 struct mcx_cmd_query_flow_group_mb_in { 2129 uint8_t cmd_table_type; 2130 uint8_t cmd_reserved0[3]; 2131 uint32_t cmd_table_id; 2132 uint32_t cmd_group_id; 2133 uint8_t cmd_reserved1[36]; 2134 } __packed __aligned(4); 2135 2136 struct mcx_cmd_query_flow_group_out { 2137 uint8_t cmd_status; 2138 uint8_t cmd_reserved0[3]; 2139 uint32_t cmd_syndrome; 2140 uint8_t cmd_reserved1[8]; 2141 } __packed __aligned(4); 2142 2143 struct mcx_cmd_query_flow_group_mb_out { 2144 uint8_t cmd_reserved0[12]; 2145 uint32_t cmd_start_flow_index; 2146 uint8_t cmd_reserved1[4]; 2147 uint32_t cmd_end_flow_index; 2148 uint8_t cmd_reserved2[20]; 2149 uint32_t cmd_match_criteria_enable; 2150 uint8_t cmd_match_criteria[512]; 2151 uint8_t cmd_reserved4[448]; 2152 } __packed __aligned(4); 2153 2154 struct mcx_cmd_query_flow_table_in { 2155 uint16_t cmd_opcode; 2156 uint8_t cmd_reserved0[4]; 2157 uint16_t cmd_op_mod; 2158 uint8_t cmd_reserved1[8]; 2159 } __packed __aligned(4); 2160 2161 struct mcx_cmd_query_flow_table_mb_in { 2162 uint8_t cmd_table_type; 2163 uint8_t cmd_reserved0[3]; 2164 uint32_t cmd_table_id; 2165 uint8_t cmd_reserved1[40]; 2166 } __packed __aligned(4); 2167 2168 struct mcx_cmd_query_flow_table_out { 2169 uint8_t cmd_status; 2170 uint8_t cmd_reserved0[3]; 2171 uint32_t cmd_syndrome; 2172 uint8_t cmd_reserved1[8]; 2173 } __packed __aligned(4); 2174 2175 struct mcx_cmd_query_flow_table_mb_out { 2176 uint8_t cmd_reserved0[4]; 2177 struct mcx_flow_table_ctx cmd_ctx; 2178 } __packed __aligned(4); 2179 2180 struct mcx_cmd_alloc_flow_counter_in { 2181 uint16_t cmd_opcode; 2182 uint8_t cmd_reserved0[4]; 2183 uint16_t cmd_op_mod; 2184 uint8_t cmd_reserved1[8]; 2185 } __packed __aligned(4); 2186 2187 struct mcx_cmd_query_rq_in { 2188 uint16_t cmd_opcode; 2189 uint8_t cmd_reserved0[4]; 2190 uint16_t cmd_op_mod; 2191 uint32_t cmd_rqn; 2192 uint8_t cmd_reserved1[4]; 2193 } __packed __aligned(4); 2194 2195 struct mcx_cmd_query_rq_out { 2196 uint8_t cmd_status; 2197 uint8_t cmd_reserved0[3]; 2198 uint32_t cmd_syndrome; 2199 uint8_t cmd_reserved1[8]; 2200 } __packed __aligned(4); 2201 2202 struct mcx_cmd_query_rq_mb_out { 2203 uint8_t cmd_reserved0[16]; 2204 struct mcx_rq_ctx cmd_ctx; 2205 }; 2206 2207 struct mcx_cmd_query_sq_in { 2208 uint16_t cmd_opcode; 2209 uint8_t cmd_reserved0[4]; 2210 uint16_t cmd_op_mod; 2211 uint32_t cmd_sqn; 2212 uint8_t cmd_reserved1[4]; 2213 } __packed __aligned(4); 2214 2215 struct mcx_cmd_query_sq_out { 2216 uint8_t cmd_status; 2217 uint8_t cmd_reserved0[3]; 2218 uint32_t cmd_syndrome; 2219 uint8_t cmd_reserved1[8]; 2220 } __packed __aligned(4); 2221 2222 struct mcx_cmd_query_sq_mb_out { 2223 uint8_t cmd_reserved0[16]; 2224 struct mcx_sq_ctx cmd_ctx; 2225 }; 2226 2227 struct mcx_cmd_alloc_flow_counter_out { 2228 uint8_t cmd_status; 2229 uint8_t cmd_reserved0[3]; 2230 uint32_t cmd_syndrome; 2231 uint8_t cmd_reserved1[2]; 2232 uint16_t cmd_flow_counter_id; 2233 uint8_t cmd_reserved2[4]; 2234 } __packed __aligned(4); 2235 2236 struct mcx_wq_doorbell { 2237 uint32_t db_recv_counter; 2238 uint32_t db_send_counter; 2239 } __packed __aligned(8); 2240 2241 struct mcx_dmamem { 2242 bus_dmamap_t mxm_map; 2243 bus_dma_segment_t mxm_seg; 2244 int mxm_nsegs; 2245 size_t mxm_size; 2246 caddr_t mxm_kva; 2247 }; 2248 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2249 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2250 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2251 #define MCX_DMA_OFF(_mxm, _off) ((void *)((_mxm)->mxm_kva + (_off))) 2252 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2253 2254 struct mcx_hwmem { 2255 bus_dmamap_t mhm_map; 2256 bus_dma_segment_t *mhm_segs; 2257 unsigned int mhm_seg_count; 2258 unsigned int mhm_npages; 2259 }; 2260 2261 struct mcx_slot { 2262 bus_dmamap_t ms_map; 2263 struct mbuf *ms_m; 2264 }; 2265 2266 struct mcx_eq { 2267 int eq_n; 2268 uint32_t eq_cons; 2269 struct mcx_dmamem eq_mem; 2270 }; 2271 2272 struct mcx_cq { 2273 int cq_n; 2274 struct mcx_dmamem cq_mem; 2275 bus_addr_t cq_doorbell; 2276 uint32_t cq_cons; 2277 uint32_t cq_count; 2278 }; 2279 2280 struct mcx_calibration { 2281 uint64_t c_timestamp; /* previous mcx chip time */ 2282 uint64_t c_uptime; /* previous kernel nanouptime */ 2283 uint64_t c_tbase; /* mcx chip time */ 2284 uint64_t c_ubase; /* kernel nanouptime */ 2285 uint64_t c_ratio; 2286 }; 2287 2288 #define MCX_CALIBRATE_FIRST 2 2289 #define MCX_CALIBRATE_NORMAL 32 2290 2291 struct mcx_rx { 2292 struct mcx_softc *rx_softc; 2293 struct ifiqueue *rx_ifiq; 2294 2295 int rx_rqn; 2296 struct mcx_dmamem rx_rq_mem; 2297 struct mcx_slot *rx_slots; 2298 bus_addr_t rx_doorbell; 2299 2300 uint32_t rx_prod; 2301 struct timeout rx_refill; 2302 struct if_rxring rx_rxr; 2303 } __aligned(64); 2304 2305 struct mcx_tx { 2306 struct mcx_softc *tx_softc; 2307 struct ifqueue *tx_ifq; 2308 2309 int tx_uar; 2310 int tx_sqn; 2311 struct mcx_dmamem tx_sq_mem; 2312 struct mcx_slot *tx_slots; 2313 bus_addr_t tx_doorbell; 2314 int tx_bf_offset; 2315 2316 uint32_t tx_cons; 2317 uint32_t tx_prod; 2318 } __aligned(64); 2319 2320 struct mcx_queues { 2321 char q_name[16]; 2322 void *q_ihc; 2323 struct mcx_softc *q_sc; 2324 int q_uar; 2325 int q_index; 2326 struct mcx_rx q_rx; 2327 struct mcx_tx q_tx; 2328 struct mcx_cq q_cq; 2329 struct mcx_eq q_eq; 2330 #if NKSTAT > 0 2331 struct kstat *q_kstat; 2332 #endif 2333 }; 2334 2335 struct mcx_flow_group { 2336 int g_id; 2337 int g_table; 2338 int g_start; 2339 int g_size; 2340 }; 2341 2342 #define MCX_FLOW_GROUP_PROMISC 0 2343 #define MCX_FLOW_GROUP_ALLMULTI 1 2344 #define MCX_FLOW_GROUP_MAC 2 2345 #define MCX_FLOW_GROUP_RSS_L4 3 2346 #define MCX_FLOW_GROUP_RSS_L3 4 2347 #define MCX_FLOW_GROUP_RSS_NONE 5 2348 #define MCX_NUM_FLOW_GROUPS 6 2349 2350 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2351 MCX_TIR_CTX_HASH_SEL_DST_IP 2352 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2353 MCX_TIR_CTX_HASH_SEL_DPORT 2354 2355 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2356 MCX_TIR_CTX_HASH_SEL_IPV4 2357 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2358 MCX_TIR_CTX_HASH_SEL_IPV6 2359 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2360 MCX_TIR_CTX_HASH_SEL_IPV4 2361 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2362 MCX_TIR_CTX_HASH_SEL_IPV6 2363 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2364 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2365 2366 /* 2367 * There are a few different pieces involved in configuring RSS. 2368 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2369 * different rx queues based on a hash value. We only create one, because 2370 * we want to scatter any traffic we can apply RSS to across all our rx 2371 * queues. Anything else will only be delivered to the first rx queue, 2372 * which doesn't require an RQT. 2373 * 2374 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2375 * queue or an RQT, and in the latter case, specifies the set of fields 2376 * hashed, the hash function, and the hash key. We need one of these for each 2377 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2378 * and one for non-RSS traffic. 2379 * 2380 * Flow tables hold flow table entries in sequence. The first entry that 2381 * matches a packet is applied, sending the packet to either another flow 2382 * table or a TIR. We use one flow table to select packets based on 2383 * destination MAC address, and a second to apply RSS. The entries in the 2384 * first table send matching packets to the second, and the entries in the 2385 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2386 * 2387 * The flow table entry that delivers packets to an RSS TIR must include match 2388 * criteria that ensure packets delivered to the TIR include all the fields 2389 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2390 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2391 * each TIR. 2392 * 2393 * All of this is a lot more flexible than we need, and we can describe most 2394 * of the stuff we need with a simple array. 2395 * 2396 * An RSS config creates a TIR with hashing enabled on a set of fields, 2397 * pointing to either the first rx queue or the RQT containing all the rx 2398 * queues, and a flow table entry that matches on an ether type and 2399 * optionally an ip proto, that delivers packets to the TIR. 2400 */ 2401 static struct mcx_rss_rule { 2402 int hash_sel; 2403 int flow_group; 2404 int ethertype; 2405 int ip_proto; 2406 } mcx_rss_config[] = { 2407 /* udp and tcp for v4/v6 */ 2408 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2409 ETHERTYPE_IP, IPPROTO_TCP }, 2410 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2411 ETHERTYPE_IPV6, IPPROTO_TCP }, 2412 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2413 ETHERTYPE_IP, IPPROTO_UDP }, 2414 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2415 ETHERTYPE_IPV6, IPPROTO_UDP }, 2416 2417 /* other v4/v6 */ 2418 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2419 ETHERTYPE_IP, 0 }, 2420 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2421 ETHERTYPE_IPV6, 0 }, 2422 2423 /* non v4/v6 */ 2424 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2425 }; 2426 2427 struct mcx_softc { 2428 struct device sc_dev; 2429 struct arpcom sc_ac; 2430 struct ifmedia sc_media; 2431 uint64_t sc_media_status; 2432 uint64_t sc_media_active; 2433 2434 pci_chipset_tag_t sc_pc; 2435 pci_intr_handle_t sc_ih; 2436 void *sc_ihc; 2437 pcitag_t sc_tag; 2438 2439 bus_dma_tag_t sc_dmat; 2440 bus_space_tag_t sc_memt; 2441 bus_space_handle_t sc_memh; 2442 bus_size_t sc_mems; 2443 2444 struct mcx_dmamem sc_cmdq_mem; 2445 unsigned int sc_cmdq_mask; 2446 unsigned int sc_cmdq_size; 2447 2448 unsigned int sc_cmdq_token; 2449 struct mutex sc_cmdq_mtx; 2450 struct rwlock sc_cmdq_kstat_lk; 2451 struct rwlock sc_cmdq_ioctl_lk; 2452 2453 struct mcx_hwmem sc_boot_pages; 2454 struct mcx_hwmem sc_init_pages; 2455 struct mcx_hwmem sc_regular_pages; 2456 2457 int sc_uar; 2458 int sc_pd; 2459 int sc_tdomain; 2460 uint32_t sc_lkey; 2461 int sc_tis; 2462 int sc_tir[nitems(mcx_rss_config)]; 2463 int sc_rqt; 2464 2465 struct mcx_dmamem sc_doorbell_mem; 2466 2467 struct mcx_eq sc_admin_eq; 2468 struct mcx_eq sc_queue_eq; 2469 2470 int sc_hardmtu; 2471 int sc_rxbufsz; 2472 2473 int sc_bf_size; 2474 int sc_max_rqt_size; 2475 2476 struct task sc_port_change; 2477 2478 int sc_mac_flow_table_id; 2479 int sc_rss_flow_table_id; 2480 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2481 int sc_promisc_flow_enabled; 2482 int sc_allmulti_flow_enabled; 2483 int sc_mcast_flow_base; 2484 int sc_extra_mcast; 2485 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2486 2487 struct mcx_calibration sc_calibration[2]; 2488 unsigned int sc_calibration_gen; 2489 struct timeout sc_calibrate; 2490 uint32_t sc_mhz; 2491 uint32_t sc_khz; 2492 2493 struct intrmap *sc_intrmap; 2494 struct mcx_queues *sc_queues; 2495 2496 int sc_mcam_reg; 2497 2498 #if NKSTAT > 0 2499 struct kstat *sc_kstat_ieee8023; 2500 struct kstat *sc_kstat_rfc2863; 2501 struct kstat *sc_kstat_rfc2819; 2502 struct kstat *sc_kstat_rfc3635; 2503 unsigned int sc_kstat_mtmp_count; 2504 struct kstat **sc_kstat_mtmp; 2505 #endif 2506 2507 struct timecounter sc_timecounter; 2508 }; 2509 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2510 2511 static int mcx_match(struct device *, void *, void *); 2512 static void mcx_attach(struct device *, struct device *, void *); 2513 2514 #if NKSTAT > 0 2515 static void mcx_kstat_attach(struct mcx_softc *); 2516 #endif 2517 2518 static void mcx_timecounter_attach(struct mcx_softc *); 2519 2520 static int mcx_version(struct mcx_softc *); 2521 static int mcx_init_wait(struct mcx_softc *); 2522 static int mcx_enable_hca(struct mcx_softc *); 2523 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2524 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2525 int, enum mcx_cmdq_slot); 2526 static int mcx_issi(struct mcx_softc *); 2527 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2528 static int mcx_hca_max_caps(struct mcx_softc *); 2529 static int mcx_hca_set_caps(struct mcx_softc *); 2530 static int mcx_init_hca(struct mcx_softc *); 2531 static int mcx_set_driver_version(struct mcx_softc *); 2532 static int mcx_iff(struct mcx_softc *); 2533 static int mcx_alloc_uar(struct mcx_softc *, int *); 2534 static int mcx_alloc_pd(struct mcx_softc *); 2535 static int mcx_alloc_tdomain(struct mcx_softc *); 2536 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2537 uint64_t, int); 2538 static int mcx_query_nic_vport_context(struct mcx_softc *); 2539 static int mcx_query_special_contexts(struct mcx_softc *); 2540 static int mcx_set_port_mtu(struct mcx_softc *, int); 2541 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2542 int); 2543 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2544 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2545 int); 2546 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2547 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2548 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2549 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2550 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2551 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2552 int *); 2553 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2554 int *); 2555 static int mcx_destroy_tir(struct mcx_softc *, int); 2556 static int mcx_create_tis(struct mcx_softc *, int *); 2557 static int mcx_destroy_tis(struct mcx_softc *, int); 2558 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2559 static int mcx_destroy_rqt(struct mcx_softc *, int); 2560 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2561 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2562 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2563 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2564 int, int, struct mcx_flow_match *); 2565 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2566 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2567 uint8_t *, uint32_t); 2568 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2569 int, int, uint32_t); 2570 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2571 2572 #if NKSTAT > 0 2573 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *); 2574 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *); 2575 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *); 2576 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *); 2577 #endif 2578 2579 #if 0 2580 static int mcx_dump_flow_table(struct mcx_softc *, int); 2581 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2582 static int mcx_dump_flow_group(struct mcx_softc *, int); 2583 #endif 2584 2585 2586 /* 2587 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2588 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2589 */ 2590 static void mcx_refill(void *); 2591 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2592 struct mcx_cq_entry *, struct mbuf_list *, 2593 const struct mcx_calibration *); 2594 static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2595 struct mcx_cq_entry *); 2596 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2597 struct mcx_cq *); 2598 2599 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2600 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2601 static int mcx_admin_intr(void *); 2602 static int mcx_cq_intr(void *); 2603 2604 static int mcx_up(struct mcx_softc *); 2605 static void mcx_down(struct mcx_softc *); 2606 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2607 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2608 static void mcx_start(struct ifqueue *); 2609 static void mcx_watchdog(struct ifnet *); 2610 static void mcx_media_add_types(struct mcx_softc *); 2611 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2612 static int mcx_media_change(struct ifnet *); 2613 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2614 static void mcx_port_change(void *); 2615 2616 static void mcx_calibrate_first(struct mcx_softc *); 2617 static void mcx_calibrate(void *); 2618 2619 static inline uint32_t 2620 mcx_rd(struct mcx_softc *, bus_size_t); 2621 static inline void 2622 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2623 static inline void 2624 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2625 2626 static uint64_t mcx_timer(struct mcx_softc *); 2627 2628 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2629 bus_size_t, u_int align); 2630 static void mcx_dmamem_zero(struct mcx_dmamem *); 2631 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2632 2633 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2634 unsigned int); 2635 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2636 2637 struct cfdriver mcx_cd = { 2638 NULL, 2639 "mcx", 2640 DV_IFNET, 2641 }; 2642 2643 const struct cfattach mcx_ca = { 2644 sizeof(struct mcx_softc), 2645 mcx_match, 2646 mcx_attach, 2647 }; 2648 2649 static const struct pci_matchid mcx_devices[] = { 2650 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2651 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2652 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2653 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2654 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2655 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2656 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2657 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2658 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 }, 2659 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2660 }; 2661 2662 struct mcx_eth_proto_capability { 2663 uint64_t cap_media; 2664 uint64_t cap_baudrate; 2665 }; 2666 2667 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2668 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2669 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2670 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2671 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2672 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2673 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2674 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2675 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2676 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2677 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2678 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2679 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2680 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2681 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2682 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2683 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2684 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2685 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2686 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2687 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2688 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2689 }; 2690 2691 static int 2692 mcx_get_id(uint32_t val) 2693 { 2694 return betoh32(val) & 0x00ffffff; 2695 } 2696 2697 static int 2698 mcx_match(struct device *parent, void *match, void *aux) 2699 { 2700 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2701 } 2702 2703 void 2704 mcx_attach(struct device *parent, struct device *self, void *aux) 2705 { 2706 struct mcx_softc *sc = (struct mcx_softc *)self; 2707 struct ifnet *ifp = &sc->sc_ac.ac_if; 2708 struct pci_attach_args *pa = aux; 2709 pcireg_t memtype; 2710 uint32_t r; 2711 unsigned int cq_stride; 2712 unsigned int cq_size; 2713 const char *intrstr; 2714 int i, msix; 2715 2716 sc->sc_pc = pa->pa_pc; 2717 sc->sc_tag = pa->pa_tag; 2718 sc->sc_dmat = pa->pa_dmat; 2719 2720 /* Map the PCI memory space */ 2721 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2722 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2723 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2724 NULL, &sc->sc_mems, 0)) { 2725 printf(": unable to map register memory\n"); 2726 return; 2727 } 2728 2729 if (mcx_version(sc) != 0) { 2730 /* error printed by mcx_version */ 2731 goto unmap; 2732 } 2733 2734 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2735 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2736 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2737 if (cq_size > MCX_MAX_CQE) { 2738 printf(", command queue size overflow %u\n", cq_size); 2739 goto unmap; 2740 } 2741 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2742 printf(", command queue entry size underflow %u\n", cq_stride); 2743 goto unmap; 2744 } 2745 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2746 printf(", command queue page overflow\n"); 2747 goto unmap; 2748 } 2749 2750 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2751 MCX_PAGE_SIZE) != 0) { 2752 printf(", unable to allocate doorbell memory\n"); 2753 goto unmap; 2754 } 2755 2756 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2757 MCX_PAGE_SIZE) != 0) { 2758 printf(", unable to allocate command queue\n"); 2759 goto dbfree; 2760 } 2761 2762 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2763 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2764 BUS_SPACE_BARRIER_WRITE); 2765 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2766 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2767 BUS_SPACE_BARRIER_WRITE); 2768 2769 if (mcx_init_wait(sc) != 0) { 2770 printf(", timeout waiting for init\n"); 2771 goto cqfree; 2772 } 2773 2774 sc->sc_cmdq_mask = cq_size - 1; 2775 sc->sc_cmdq_size = cq_stride; 2776 rw_init(&sc->sc_cmdq_kstat_lk, "mcxkstat"); 2777 rw_init(&sc->sc_cmdq_ioctl_lk, "mcxioctl"); 2778 mtx_init(&sc->sc_cmdq_mtx, IPL_NET); 2779 2780 if (mcx_enable_hca(sc) != 0) { 2781 /* error printed by mcx_enable_hca */ 2782 goto cqfree; 2783 } 2784 2785 if (mcx_issi(sc) != 0) { 2786 /* error printed by mcx_issi */ 2787 goto teardown; 2788 } 2789 2790 if (mcx_pages(sc, &sc->sc_boot_pages, 2791 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2792 /* error printed by mcx_pages */ 2793 goto teardown; 2794 } 2795 2796 if (mcx_hca_max_caps(sc) != 0) { 2797 /* error printed by mcx_hca_max_caps */ 2798 goto teardown; 2799 } 2800 2801 if (mcx_hca_set_caps(sc) != 0) { 2802 /* error printed by mcx_hca_set_caps */ 2803 goto teardown; 2804 } 2805 2806 if (mcx_pages(sc, &sc->sc_init_pages, 2807 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2808 /* error printed by mcx_pages */ 2809 goto teardown; 2810 } 2811 2812 if (mcx_init_hca(sc) != 0) { 2813 /* error printed by mcx_init_hca */ 2814 goto teardown; 2815 } 2816 2817 if (mcx_pages(sc, &sc->sc_regular_pages, 2818 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2819 /* error printed by mcx_pages */ 2820 goto teardown; 2821 } 2822 2823 /* apparently not necessary? */ 2824 if (mcx_set_driver_version(sc) != 0) { 2825 /* error printed by mcx_set_driver_version */ 2826 goto teardown; 2827 } 2828 2829 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2830 /* error printed by mcx_iff? */ 2831 goto teardown; 2832 } 2833 2834 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2835 /* error printed by mcx_alloc_uar */ 2836 goto teardown; 2837 } 2838 2839 if (mcx_alloc_pd(sc) != 0) { 2840 /* error printed by mcx_alloc_pd */ 2841 goto teardown; 2842 } 2843 2844 if (mcx_alloc_tdomain(sc) != 0) { 2845 /* error printed by mcx_alloc_tdomain */ 2846 goto teardown; 2847 } 2848 2849 msix = pci_intr_msix_count(pa); 2850 if (msix < 2) { 2851 printf(": not enough msi-x vectors\n"); 2852 goto teardown; 2853 } 2854 2855 /* 2856 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2857 * mellanox support tells me legacy interrupts are not supported, 2858 * so we're stuck with just msi-x. 2859 */ 2860 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2861 printf(": unable to map interrupt\n"); 2862 goto teardown; 2863 } 2864 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2865 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2866 IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc)); 2867 if (sc->sc_ihc == NULL) { 2868 printf(": unable to establish interrupt"); 2869 if (intrstr != NULL) 2870 printf(" at %s", intrstr); 2871 printf("\n"); 2872 goto teardown; 2873 } 2874 2875 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2876 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2877 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2878 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2879 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2880 /* error printed by mcx_create_eq */ 2881 goto teardown; 2882 } 2883 2884 if (mcx_query_nic_vport_context(sc) != 0) { 2885 /* error printed by mcx_query_nic_vport_context */ 2886 goto teardown; 2887 } 2888 2889 if (mcx_query_special_contexts(sc) != 0) { 2890 /* error printed by mcx_query_special_contexts */ 2891 goto teardown; 2892 } 2893 2894 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2895 /* error printed by mcx_set_port_mtu */ 2896 goto teardown; 2897 } 2898 2899 printf(", %s, address %s\n", intrstr, 2900 ether_sprintf(sc->sc_ac.ac_enaddr)); 2901 2902 msix--; /* admin ops took one */ 2903 sc->sc_intrmap = intrmap_create(&sc->sc_dev, msix, MCX_MAX_QUEUES, 2904 INTRMAP_POWEROF2); 2905 if (sc->sc_intrmap == NULL) { 2906 printf("%s: unable to create interrupt map\n", DEVNAME(sc)); 2907 goto teardown; 2908 } 2909 sc->sc_queues = mallocarray(intrmap_count(sc->sc_intrmap), 2910 sizeof(*sc->sc_queues), M_DEVBUF, M_WAITOK|M_ZERO); 2911 if (sc->sc_queues == NULL) { 2912 printf("%s: unable to create queues\n", DEVNAME(sc)); 2913 goto intrunmap; 2914 } 2915 2916 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2917 ifp->if_softc = sc; 2918 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2919 ifp->if_xflags = IFXF_MPSAFE; 2920 ifp->if_ioctl = mcx_ioctl; 2921 ifp->if_qstart = mcx_start; 2922 ifp->if_watchdog = mcx_watchdog; 2923 ifp->if_hardmtu = sc->sc_hardmtu; 2924 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 | 2925 IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 | 2926 IFCAP_CSUM_TCPv6; 2927 #if NVLAN > 0 2928 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2929 #endif 2930 ifq_set_maxlen(&ifp->if_snd, 1024); 2931 2932 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2933 mcx_media_status); 2934 mcx_media_add_types(sc); 2935 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2936 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2937 2938 if_attach(ifp); 2939 ether_ifattach(ifp); 2940 2941 if_attach_iqueues(ifp, intrmap_count(sc->sc_intrmap)); 2942 if_attach_queues(ifp, intrmap_count(sc->sc_intrmap)); 2943 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 2944 struct ifiqueue *ifiq = ifp->if_iqs[i]; 2945 struct ifqueue *ifq = ifp->if_ifqs[i]; 2946 struct mcx_queues *q = &sc->sc_queues[i]; 2947 struct mcx_rx *rx = &q->q_rx; 2948 struct mcx_tx *tx = &q->q_tx; 2949 pci_intr_handle_t ih; 2950 int vec; 2951 2952 vec = i + 1; 2953 q->q_sc = sc; 2954 q->q_index = i; 2955 2956 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 2957 printf("%s: unable to alloc uar %d\n", 2958 DEVNAME(sc), i); 2959 goto intrdisestablish; 2960 } 2961 2962 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 2963 printf("%s: unable to create event queue %d\n", 2964 DEVNAME(sc), i); 2965 goto intrdisestablish; 2966 } 2967 2968 rx->rx_softc = sc; 2969 rx->rx_ifiq = ifiq; 2970 timeout_set(&rx->rx_refill, mcx_refill, rx); 2971 ifiq->ifiq_softc = rx; 2972 2973 tx->tx_softc = sc; 2974 tx->tx_ifq = ifq; 2975 ifq->ifq_softc = tx; 2976 2977 if (pci_intr_map_msix(pa, vec, &ih) != 0) { 2978 printf("%s: unable to map queue interrupt %d\n", 2979 DEVNAME(sc), i); 2980 goto intrdisestablish; 2981 } 2982 snprintf(q->q_name, sizeof(q->q_name), "%s:%d", 2983 DEVNAME(sc), i); 2984 q->q_ihc = pci_intr_establish_cpu(sc->sc_pc, ih, 2985 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 2986 mcx_cq_intr, q, q->q_name); 2987 if (q->q_ihc == NULL) { 2988 printf("%s: unable to establish interrupt %d\n", 2989 DEVNAME(sc), i); 2990 goto intrdisestablish; 2991 } 2992 } 2993 2994 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2995 2996 task_set(&sc->sc_port_change, mcx_port_change, sc); 2997 mcx_port_change(sc); 2998 2999 sc->sc_mac_flow_table_id = -1; 3000 sc->sc_rss_flow_table_id = -1; 3001 sc->sc_rqt = -1; 3002 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 3003 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 3004 mfg->g_id = -1; 3005 mfg->g_table = -1; 3006 mfg->g_size = 0; 3007 mfg->g_start = 0; 3008 } 3009 sc->sc_extra_mcast = 0; 3010 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 3011 3012 #if NKSTAT > 0 3013 mcx_kstat_attach(sc); 3014 #endif 3015 mcx_timecounter_attach(sc); 3016 return; 3017 3018 intrdisestablish: 3019 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 3020 struct mcx_queues *q = &sc->sc_queues[i]; 3021 if (q->q_ihc == NULL) 3022 continue; 3023 pci_intr_disestablish(sc->sc_pc, q->q_ihc); 3024 q->q_ihc = NULL; 3025 } 3026 free(sc->sc_queues, M_DEVBUF, 3027 intrmap_count(sc->sc_intrmap) * sizeof(*sc->sc_queues)); 3028 intrunmap: 3029 intrmap_destroy(sc->sc_intrmap); 3030 sc->sc_intrmap = NULL; 3031 teardown: 3032 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 3033 /* error printed by mcx_teardown_hca, and we're already unwinding */ 3034 cqfree: 3035 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 3036 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 3037 BUS_SPACE_BARRIER_WRITE); 3038 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 3039 MCX_CMDQ_INTERFACE_DISABLED); 3040 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 3041 BUS_SPACE_BARRIER_WRITE); 3042 3043 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 3044 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 3045 BUS_SPACE_BARRIER_WRITE); 3046 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 3047 3048 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 3049 dbfree: 3050 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 3051 unmap: 3052 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 3053 sc->sc_mems = 0; 3054 } 3055 3056 static int 3057 mcx_version(struct mcx_softc *sc) 3058 { 3059 uint32_t fw0, fw1; 3060 uint16_t cmdif; 3061 3062 fw0 = mcx_rd(sc, MCX_FW_VER); 3063 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 3064 3065 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 3066 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 3067 3068 cmdif = MCX_CMDIF(fw1); 3069 if (cmdif != MCX_CMD_IF_SUPPORTED) { 3070 printf(", unsupported command interface %u\n", cmdif); 3071 return (-1); 3072 } 3073 3074 return (0); 3075 } 3076 3077 static int 3078 mcx_init_wait(struct mcx_softc *sc) 3079 { 3080 unsigned int i; 3081 uint32_t r; 3082 3083 for (i = 0; i < 2000; i++) { 3084 r = mcx_rd(sc, MCX_STATE); 3085 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 3086 return (0); 3087 3088 delay(1000); 3089 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 3090 BUS_SPACE_BARRIER_READ); 3091 } 3092 3093 return (-1); 3094 } 3095 3096 static uint8_t 3097 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3098 unsigned int msec) 3099 { 3100 unsigned int i; 3101 3102 for (i = 0; i < msec; i++) { 3103 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3104 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3105 3106 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 3107 MCX_CQ_STATUS_OWN_SW) 3108 return (0); 3109 3110 delay(1000); 3111 } 3112 3113 return (ETIMEDOUT); 3114 } 3115 3116 static uint32_t 3117 mcx_mix_u64(uint32_t xor, uint64_t u64) 3118 { 3119 xor ^= u64 >> 32; 3120 xor ^= u64; 3121 3122 return (xor); 3123 } 3124 3125 static uint32_t 3126 mcx_mix_u32(uint32_t xor, uint32_t u32) 3127 { 3128 xor ^= u32; 3129 3130 return (xor); 3131 } 3132 3133 static uint32_t 3134 mcx_mix_u8(uint32_t xor, uint8_t u8) 3135 { 3136 xor ^= u8; 3137 3138 return (xor); 3139 } 3140 3141 static uint8_t 3142 mcx_mix_done(uint32_t xor) 3143 { 3144 xor ^= xor >> 16; 3145 xor ^= xor >> 8; 3146 3147 return (xor); 3148 } 3149 3150 static uint8_t 3151 mcx_xor(const void *buf, size_t len) 3152 { 3153 const uint32_t *dwords = buf; 3154 uint32_t xor = 0xff; 3155 size_t i; 3156 3157 len /= sizeof(*dwords); 3158 3159 for (i = 0; i < len; i++) 3160 xor ^= dwords[i]; 3161 3162 return (mcx_mix_done(xor)); 3163 } 3164 3165 static uint8_t 3166 mcx_cmdq_token(struct mcx_softc *sc) 3167 { 3168 uint8_t token; 3169 3170 mtx_enter(&sc->sc_cmdq_mtx); 3171 do { 3172 token = ++sc->sc_cmdq_token; 3173 } while (token == 0); 3174 mtx_leave(&sc->sc_cmdq_mtx); 3175 3176 return (token); 3177 } 3178 3179 static struct mcx_cmdq_entry * 3180 mcx_get_cmdq_entry(struct mcx_softc *sc, enum mcx_cmdq_slot slot) 3181 { 3182 struct mcx_cmdq_entry *cqe; 3183 3184 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3185 cqe += slot; 3186 3187 /* make sure the slot isn't running a command already */ 3188 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3189 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3190 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) != 3191 MCX_CQ_STATUS_OWN_SW) 3192 cqe = NULL; 3193 3194 return (cqe); 3195 } 3196 3197 static void 3198 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3199 uint32_t ilen, uint32_t olen, uint8_t token) 3200 { 3201 memset(cqe, 0, sc->sc_cmdq_size); 3202 3203 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 3204 htobem32(&cqe->cq_input_length, ilen); 3205 htobem32(&cqe->cq_output_length, olen); 3206 cqe->cq_token = token; 3207 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 3208 } 3209 3210 static void 3211 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 3212 { 3213 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 3214 } 3215 3216 static int 3217 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 3218 { 3219 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 3220 return (0); 3221 } 3222 3223 static void * 3224 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 3225 { 3226 return (&cqe->cq_input_data); 3227 } 3228 3229 static void * 3230 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3231 { 3232 return (&cqe->cq_output_data); 3233 } 3234 3235 static void 3236 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3237 unsigned int slot) 3238 { 3239 mcx_cmdq_sign(cqe); 3240 3241 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3242 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3243 3244 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3245 mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t), 3246 BUS_SPACE_BARRIER_WRITE); 3247 } 3248 3249 static int 3250 mcx_cmdq_exec(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3251 unsigned int slot, unsigned int msec) 3252 { 3253 int err; 3254 3255 if (slot == MCX_CMDQ_SLOT_POLL) { 3256 mcx_cmdq_post(sc, cqe, slot); 3257 return (mcx_cmdq_poll(sc, cqe, msec)); 3258 } 3259 3260 mtx_enter(&sc->sc_cmdq_mtx); 3261 mcx_cmdq_post(sc, cqe, slot); 3262 3263 err = 0; 3264 while (err == 0) { 3265 err = msleep_nsec(&sc->sc_cmdq_token, &sc->sc_cmdq_mtx, 0, 3266 "mcxcmd", msec * 1000); 3267 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 0, 3268 MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3269 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 3270 MCX_CQ_STATUS_OWN_SW) { 3271 err = 0; 3272 break; 3273 } 3274 } 3275 3276 mtx_leave(&sc->sc_cmdq_mtx); 3277 return (err); 3278 } 3279 3280 static int 3281 mcx_enable_hca(struct mcx_softc *sc) 3282 { 3283 struct mcx_cmdq_entry *cqe; 3284 struct mcx_cmd_enable_hca_in *in; 3285 struct mcx_cmd_enable_hca_out *out; 3286 int error; 3287 uint8_t status; 3288 3289 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3290 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3291 3292 in = mcx_cmdq_in(cqe); 3293 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3294 in->cmd_op_mod = htobe16(0); 3295 in->cmd_function_id = htobe16(0); 3296 3297 mcx_cmdq_post(sc, cqe, 0); 3298 3299 error = mcx_cmdq_poll(sc, cqe, 1000); 3300 if (error != 0) { 3301 printf(", hca enable timeout\n"); 3302 return (-1); 3303 } 3304 if (mcx_cmdq_verify(cqe) != 0) { 3305 printf(", hca enable command corrupt\n"); 3306 return (-1); 3307 } 3308 3309 status = cqe->cq_output_data[0]; 3310 if (status != MCX_CQ_STATUS_OK) { 3311 printf(", hca enable failed (%x)\n", status); 3312 return (-1); 3313 } 3314 3315 return (0); 3316 } 3317 3318 static int 3319 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3320 { 3321 struct mcx_cmdq_entry *cqe; 3322 struct mcx_cmd_teardown_hca_in *in; 3323 struct mcx_cmd_teardown_hca_out *out; 3324 int error; 3325 uint8_t status; 3326 3327 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3328 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3329 3330 in = mcx_cmdq_in(cqe); 3331 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3332 in->cmd_op_mod = htobe16(0); 3333 in->cmd_profile = profile; 3334 3335 mcx_cmdq_post(sc, cqe, 0); 3336 3337 error = mcx_cmdq_poll(sc, cqe, 1000); 3338 if (error != 0) { 3339 printf(", hca teardown timeout\n"); 3340 return (-1); 3341 } 3342 if (mcx_cmdq_verify(cqe) != 0) { 3343 printf(", hca teardown command corrupt\n"); 3344 return (-1); 3345 } 3346 3347 status = cqe->cq_output_data[0]; 3348 if (status != MCX_CQ_STATUS_OK) { 3349 printf(", hca teardown failed (%x)\n", status); 3350 return (-1); 3351 } 3352 3353 return (0); 3354 } 3355 3356 static int 3357 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3358 unsigned int nmb, uint64_t *ptr, uint8_t token) 3359 { 3360 caddr_t kva; 3361 uint64_t dva; 3362 int i; 3363 int error; 3364 3365 error = mcx_dmamem_alloc(sc, mxm, 3366 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3367 if (error != 0) 3368 return (error); 3369 3370 mcx_dmamem_zero(mxm); 3371 3372 dva = MCX_DMA_DVA(mxm); 3373 kva = MCX_DMA_KVA(mxm); 3374 for (i = 0; i < nmb; i++) { 3375 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3376 3377 /* patch the cqe or mbox pointing at this one */ 3378 htobem64(ptr, dva); 3379 3380 /* fill in this mbox */ 3381 htobem32(&mbox->mb_block_number, i); 3382 mbox->mb_token = token; 3383 3384 /* move to the next one */ 3385 ptr = &mbox->mb_next_ptr; 3386 3387 dva += MCX_CMDQ_MAILBOX_SIZE; 3388 kva += MCX_CMDQ_MAILBOX_SIZE; 3389 } 3390 3391 return (0); 3392 } 3393 3394 static uint32_t 3395 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3396 { 3397 uint32_t xor = 0xff; 3398 3399 /* only 3 fields get set, so mix them directly */ 3400 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3401 xor = mcx_mix_u32(xor, mb->mb_block_number); 3402 xor = mcx_mix_u8(xor, mb->mb_token); 3403 3404 return (mcx_mix_done(xor)); 3405 } 3406 3407 static void 3408 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3409 { 3410 caddr_t kva; 3411 int i; 3412 3413 kva = MCX_DMA_KVA(mxm); 3414 3415 for (i = 0; i < nmb; i++) { 3416 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3417 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3418 mb->mb_ctrl_signature = sig; 3419 mb->mb_signature = sig ^ 3420 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3421 3422 kva += MCX_CMDQ_MAILBOX_SIZE; 3423 } 3424 } 3425 3426 static void 3427 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3428 { 3429 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3430 0, MCX_DMA_LEN(mxm), ops); 3431 } 3432 3433 static struct mcx_cmdq_mailbox * 3434 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3435 { 3436 caddr_t kva; 3437 3438 kva = MCX_DMA_KVA(mxm); 3439 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3440 3441 return ((struct mcx_cmdq_mailbox *)kva); 3442 } 3443 3444 static inline void * 3445 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3446 { 3447 return (&mb->mb_data); 3448 } 3449 3450 static void 3451 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3452 void *b, size_t len) 3453 { 3454 caddr_t buf = b; 3455 struct mcx_cmdq_mailbox *mb; 3456 int i; 3457 3458 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3459 for (i = 0; i < nmb; i++) { 3460 3461 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 3462 3463 if (sizeof(mb->mb_data) >= len) 3464 break; 3465 3466 buf += sizeof(mb->mb_data); 3467 len -= sizeof(mb->mb_data); 3468 mb++; 3469 } 3470 } 3471 3472 static void 3473 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3474 struct mcx_dmamem *buf) 3475 { 3476 uint64_t *pas; 3477 int mbox, mbox_pages, i; 3478 3479 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3480 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3481 3482 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3483 pas += (offset / sizeof(*pas)); 3484 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3485 for (i = 0; i < npages; i++) { 3486 if (i == mbox_pages) { 3487 mbox++; 3488 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3489 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3490 } 3491 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3492 pas++; 3493 } 3494 } 3495 3496 static void 3497 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3498 { 3499 caddr_t buf = b; 3500 struct mcx_cmdq_mailbox *mb; 3501 int i; 3502 3503 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3504 for (i = 0; i < nmb; i++) { 3505 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 3506 3507 if (sizeof(mb->mb_data) >= len) 3508 break; 3509 3510 buf += sizeof(mb->mb_data); 3511 len -= sizeof(mb->mb_data); 3512 mb++; 3513 } 3514 } 3515 3516 static void 3517 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3518 { 3519 mcx_dmamem_free(sc, mxm); 3520 } 3521 3522 #if 0 3523 static void 3524 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3525 { 3526 unsigned int i; 3527 3528 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3529 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 3530 3531 printf(", idata "); 3532 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3533 printf("%02x", cqe->cq_input_data[i]); 3534 3535 printf(", odata "); 3536 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3537 printf("%02x", cqe->cq_output_data[i]); 3538 3539 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3540 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 3541 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3542 } 3543 3544 static void 3545 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3546 { 3547 int i, j; 3548 uint8_t *d; 3549 3550 for (i = 0; i < num; i++) { 3551 struct mcx_cmdq_mailbox *mbox; 3552 mbox = mcx_cq_mbox(mboxes, i); 3553 3554 d = mcx_cq_mbox_data(mbox); 3555 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3556 if (j != 0 && (j % 16 == 0)) 3557 printf("\n"); 3558 printf("%.2x ", d[j]); 3559 } 3560 } 3561 } 3562 #endif 3563 3564 static int 3565 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3566 int len, enum mcx_cmdq_slot slot) 3567 { 3568 struct mcx_dmamem mxm; 3569 struct mcx_cmdq_entry *cqe; 3570 struct mcx_cmd_access_reg_in *in; 3571 struct mcx_cmd_access_reg_out *out; 3572 uint8_t token = mcx_cmdq_token(sc); 3573 int error, nmb; 3574 3575 cqe = mcx_get_cmdq_entry(sc, slot); 3576 if (cqe == NULL) 3577 return (-1); 3578 3579 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3580 token); 3581 3582 in = mcx_cmdq_in(cqe); 3583 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3584 in->cmd_op_mod = htobe16(op); 3585 in->cmd_register_id = htobe16(reg); 3586 3587 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3588 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3589 &cqe->cq_output_ptr, token) != 0) { 3590 printf(", unable to allocate access reg mailboxen\n"); 3591 return (-1); 3592 } 3593 cqe->cq_input_ptr = cqe->cq_output_ptr; 3594 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3595 mcx_cmdq_mboxes_sign(&mxm, nmb); 3596 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3597 3598 error = mcx_cmdq_exec(sc, cqe, slot, 1000); 3599 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3600 3601 if (error != 0) { 3602 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3603 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3604 goto free; 3605 } 3606 error = mcx_cmdq_verify(cqe); 3607 if (error != 0) { 3608 printf("%s: access reg (%s %x) reply corrupt\n", 3609 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3610 reg); 3611 goto free; 3612 } 3613 3614 out = mcx_cmdq_out(cqe); 3615 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3616 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3617 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3618 reg, out->cmd_status, betoh32(out->cmd_syndrome)); 3619 error = -1; 3620 goto free; 3621 } 3622 3623 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3624 free: 3625 mcx_dmamem_free(sc, &mxm); 3626 3627 return (error); 3628 } 3629 3630 static int 3631 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3632 unsigned int slot) 3633 { 3634 struct mcx_cmd_set_issi_in *in; 3635 struct mcx_cmd_set_issi_out *out; 3636 uint8_t status; 3637 3638 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3639 3640 in = mcx_cmdq_in(cqe); 3641 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3642 in->cmd_op_mod = htobe16(0); 3643 in->cmd_current_issi = htobe16(MCX_ISSI); 3644 3645 mcx_cmdq_post(sc, cqe, slot); 3646 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3647 return (-1); 3648 if (mcx_cmdq_verify(cqe) != 0) 3649 return (-1); 3650 3651 status = cqe->cq_output_data[0]; 3652 if (status != MCX_CQ_STATUS_OK) 3653 return (-1); 3654 3655 return (0); 3656 } 3657 3658 static int 3659 mcx_issi(struct mcx_softc *sc) 3660 { 3661 struct mcx_dmamem mxm; 3662 struct mcx_cmdq_entry *cqe; 3663 struct mcx_cmd_query_issi_in *in; 3664 struct mcx_cmd_query_issi_il_out *out; 3665 struct mcx_cmd_query_issi_mb_out *mb; 3666 uint8_t token = mcx_cmdq_token(sc); 3667 uint8_t status; 3668 int error; 3669 3670 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3671 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3672 3673 in = mcx_cmdq_in(cqe); 3674 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3675 in->cmd_op_mod = htobe16(0); 3676 3677 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3678 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3679 &cqe->cq_output_ptr, token) != 0) { 3680 printf(", unable to allocate query issi mailbox\n"); 3681 return (-1); 3682 } 3683 mcx_cmdq_mboxes_sign(&mxm, 1); 3684 3685 mcx_cmdq_post(sc, cqe, 0); 3686 error = mcx_cmdq_poll(sc, cqe, 1000); 3687 if (error != 0) { 3688 printf(", query issi timeout\n"); 3689 goto free; 3690 } 3691 error = mcx_cmdq_verify(cqe); 3692 if (error != 0) { 3693 printf(", query issi reply corrupt\n"); 3694 goto free; 3695 } 3696 3697 status = cqe->cq_output_data[0]; 3698 switch (status) { 3699 case MCX_CQ_STATUS_OK: 3700 break; 3701 case MCX_CQ_STATUS_BAD_OPCODE: 3702 /* use ISSI 0 */ 3703 goto free; 3704 default: 3705 printf(", query issi failed (%x)\n", status); 3706 error = -1; 3707 goto free; 3708 } 3709 3710 out = mcx_cmdq_out(cqe); 3711 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3712 /* use ISSI 1 */ 3713 goto free; 3714 } 3715 3716 /* don't need to read cqe anymore, can be used for SET ISSI */ 3717 3718 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3719 CTASSERT(MCX_ISSI < NBBY); 3720 /* XXX math is hard */ 3721 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3722 /* use ISSI 0 */ 3723 goto free; 3724 } 3725 3726 if (mcx_set_issi(sc, cqe, 0) != 0) { 3727 /* ignore the error, just use ISSI 0 */ 3728 } else { 3729 /* use ISSI 1 */ 3730 } 3731 3732 free: 3733 mcx_cq_mboxes_free(sc, &mxm); 3734 return (error); 3735 } 3736 3737 static int 3738 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3739 int32_t *npages, uint16_t *func_id) 3740 { 3741 struct mcx_cmdq_entry *cqe; 3742 struct mcx_cmd_query_pages_in *in; 3743 struct mcx_cmd_query_pages_out *out; 3744 3745 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3746 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3747 3748 in = mcx_cmdq_in(cqe); 3749 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3750 in->cmd_op_mod = type; 3751 3752 mcx_cmdq_post(sc, cqe, 0); 3753 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3754 printf(", query pages timeout\n"); 3755 return (-1); 3756 } 3757 if (mcx_cmdq_verify(cqe) != 0) { 3758 printf(", query pages reply corrupt\n"); 3759 return (-1); 3760 } 3761 3762 out = mcx_cmdq_out(cqe); 3763 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3764 printf(", query pages failed (%x)\n", out->cmd_status); 3765 return (-1); 3766 } 3767 3768 *func_id = out->cmd_func_id; 3769 *npages = bemtoh32(&out->cmd_num_pages); 3770 3771 return (0); 3772 } 3773 3774 struct bus_dma_iter { 3775 bus_dmamap_t i_map; 3776 bus_size_t i_offset; 3777 unsigned int i_index; 3778 }; 3779 3780 static void 3781 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3782 { 3783 i->i_map = map; 3784 i->i_offset = 0; 3785 i->i_index = 0; 3786 } 3787 3788 static bus_addr_t 3789 bus_dma_iter_addr(struct bus_dma_iter *i) 3790 { 3791 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3792 } 3793 3794 static void 3795 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3796 { 3797 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3798 bus_size_t diff; 3799 3800 do { 3801 diff = seg->ds_len - i->i_offset; 3802 if (size < diff) 3803 break; 3804 3805 size -= diff; 3806 3807 seg++; 3808 3809 i->i_offset = 0; 3810 i->i_index++; 3811 } while (size > 0); 3812 3813 i->i_offset += size; 3814 } 3815 3816 static int 3817 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3818 { 3819 struct mcx_dmamem mxm; 3820 struct mcx_cmdq_entry *cqe; 3821 struct mcx_cmd_manage_pages_in *in; 3822 struct mcx_cmd_manage_pages_out *out; 3823 unsigned int paslen, nmb, i, j, npages; 3824 struct bus_dma_iter iter; 3825 uint64_t *pas; 3826 uint8_t status; 3827 uint8_t token = mcx_cmdq_token(sc); 3828 int error; 3829 3830 npages = mhm->mhm_npages; 3831 3832 paslen = sizeof(*pas) * npages; 3833 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3834 3835 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3836 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3837 3838 in = mcx_cmdq_in(cqe); 3839 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3840 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3841 in->cmd_func_id = func_id; 3842 htobem32(&in->cmd_input_num_entries, npages); 3843 3844 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3845 &cqe->cq_input_ptr, token) != 0) { 3846 printf(", unable to allocate manage pages mailboxen\n"); 3847 return (-1); 3848 } 3849 3850 bus_dma_iter_init(&iter, mhm->mhm_map); 3851 for (i = 0; i < nmb; i++) { 3852 unsigned int lim; 3853 3854 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3855 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3856 3857 for (j = 0; j < lim; j++) { 3858 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3859 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3860 } 3861 3862 npages -= lim; 3863 } 3864 3865 mcx_cmdq_mboxes_sign(&mxm, nmb); 3866 3867 mcx_cmdq_post(sc, cqe, 0); 3868 error = mcx_cmdq_poll(sc, cqe, 1000); 3869 if (error != 0) { 3870 printf(", manage pages timeout\n"); 3871 goto free; 3872 } 3873 error = mcx_cmdq_verify(cqe); 3874 if (error != 0) { 3875 printf(", manage pages reply corrupt\n"); 3876 goto free; 3877 } 3878 3879 status = cqe->cq_output_data[0]; 3880 if (status != MCX_CQ_STATUS_OK) { 3881 printf(", manage pages failed (%x)\n", status); 3882 error = -1; 3883 goto free; 3884 } 3885 3886 free: 3887 mcx_dmamem_free(sc, &mxm); 3888 3889 return (error); 3890 } 3891 3892 static int 3893 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3894 { 3895 int32_t npages; 3896 uint16_t func_id; 3897 3898 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3899 /* error printed by mcx_query_pages */ 3900 return (-1); 3901 } 3902 3903 if (npages < 1) 3904 return (0); 3905 3906 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3907 printf(", unable to allocate hwmem\n"); 3908 return (-1); 3909 } 3910 3911 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3912 printf(", unable to add hwmem\n"); 3913 goto free; 3914 } 3915 3916 return (0); 3917 3918 free: 3919 mcx_hwmem_free(sc, mhm); 3920 3921 return (-1); 3922 } 3923 3924 static int 3925 mcx_hca_max_caps(struct mcx_softc *sc) 3926 { 3927 struct mcx_dmamem mxm; 3928 struct mcx_cmdq_entry *cqe; 3929 struct mcx_cmd_query_hca_cap_in *in; 3930 struct mcx_cmd_query_hca_cap_out *out; 3931 struct mcx_cmdq_mailbox *mb; 3932 struct mcx_cap_device *hca; 3933 uint8_t status; 3934 uint8_t token = mcx_cmdq_token(sc); 3935 int error; 3936 3937 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3938 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3939 token); 3940 3941 in = mcx_cmdq_in(cqe); 3942 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3943 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3944 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3945 3946 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3947 &cqe->cq_output_ptr, token) != 0) { 3948 printf(", unable to allocate query hca caps mailboxen\n"); 3949 return (-1); 3950 } 3951 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3952 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3953 3954 mcx_cmdq_post(sc, cqe, 0); 3955 error = mcx_cmdq_poll(sc, cqe, 1000); 3956 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3957 3958 if (error != 0) { 3959 printf(", query hca caps timeout\n"); 3960 goto free; 3961 } 3962 error = mcx_cmdq_verify(cqe); 3963 if (error != 0) { 3964 printf(", query hca caps reply corrupt\n"); 3965 goto free; 3966 } 3967 3968 status = cqe->cq_output_data[0]; 3969 if (status != MCX_CQ_STATUS_OK) { 3970 printf(", query hca caps failed (%x)\n", status); 3971 error = -1; 3972 goto free; 3973 } 3974 3975 mb = mcx_cq_mbox(&mxm, 0); 3976 hca = mcx_cq_mbox_data(mb); 3977 3978 if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE) 3979 != MCX_CAP_DEVICE_PORT_TYPE_ETH) { 3980 printf(", not in ethernet mode\n"); 3981 error = -1; 3982 goto free; 3983 } 3984 if (hca->log_pg_sz > PAGE_SHIFT) { 3985 printf(", minimum system page shift %u is too large\n", 3986 hca->log_pg_sz); 3987 error = -1; 3988 goto free; 3989 } 3990 /* 3991 * blueflame register is split into two buffers, and we must alternate 3992 * between the two of them. 3993 */ 3994 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3995 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 3996 3997 if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG) 3998 sc->sc_mcam_reg = 1; 3999 4000 sc->sc_mhz = bemtoh32(&hca->device_frequency_mhz); 4001 sc->sc_khz = bemtoh32(&hca->device_frequency_khz); 4002 4003 free: 4004 mcx_dmamem_free(sc, &mxm); 4005 4006 return (error); 4007 } 4008 4009 static int 4010 mcx_hca_set_caps(struct mcx_softc *sc) 4011 { 4012 struct mcx_dmamem mxm; 4013 struct mcx_cmdq_entry *cqe; 4014 struct mcx_cmd_query_hca_cap_in *in; 4015 struct mcx_cmd_query_hca_cap_out *out; 4016 struct mcx_cmdq_mailbox *mb; 4017 struct mcx_cap_device *hca; 4018 uint8_t status; 4019 uint8_t token = mcx_cmdq_token(sc); 4020 int error; 4021 4022 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4023 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 4024 token); 4025 4026 in = mcx_cmdq_in(cqe); 4027 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 4028 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 4029 MCX_CMD_QUERY_HCA_CAP_DEVICE); 4030 4031 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 4032 &cqe->cq_output_ptr, token) != 0) { 4033 printf(", unable to allocate manage pages mailboxen\n"); 4034 return (-1); 4035 } 4036 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 4037 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 4038 4039 mcx_cmdq_post(sc, cqe, 0); 4040 error = mcx_cmdq_poll(sc, cqe, 1000); 4041 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 4042 4043 if (error != 0) { 4044 printf(", query hca caps timeout\n"); 4045 goto free; 4046 } 4047 error = mcx_cmdq_verify(cqe); 4048 if (error != 0) { 4049 printf(", query hca caps reply corrupt\n"); 4050 goto free; 4051 } 4052 4053 status = cqe->cq_output_data[0]; 4054 if (status != MCX_CQ_STATUS_OK) { 4055 printf(", query hca caps failed (%x)\n", status); 4056 error = -1; 4057 goto free; 4058 } 4059 4060 mb = mcx_cq_mbox(&mxm, 0); 4061 hca = mcx_cq_mbox_data(mb); 4062 4063 hca->log_pg_sz = PAGE_SHIFT; 4064 4065 free: 4066 mcx_dmamem_free(sc, &mxm); 4067 4068 return (error); 4069 } 4070 4071 4072 static int 4073 mcx_init_hca(struct mcx_softc *sc) 4074 { 4075 struct mcx_cmdq_entry *cqe; 4076 struct mcx_cmd_init_hca_in *in; 4077 struct mcx_cmd_init_hca_out *out; 4078 int error; 4079 uint8_t status; 4080 4081 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4082 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4083 4084 in = mcx_cmdq_in(cqe); 4085 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 4086 in->cmd_op_mod = htobe16(0); 4087 4088 mcx_cmdq_post(sc, cqe, 0); 4089 4090 error = mcx_cmdq_poll(sc, cqe, 1000); 4091 if (error != 0) { 4092 printf(", hca init timeout\n"); 4093 return (-1); 4094 } 4095 if (mcx_cmdq_verify(cqe) != 0) { 4096 printf(", hca init command corrupt\n"); 4097 return (-1); 4098 } 4099 4100 status = cqe->cq_output_data[0]; 4101 if (status != MCX_CQ_STATUS_OK) { 4102 printf(", hca init failed (%x)\n", status); 4103 return (-1); 4104 } 4105 4106 return (0); 4107 } 4108 4109 static int 4110 mcx_set_driver_version(struct mcx_softc *sc) 4111 { 4112 struct mcx_dmamem mxm; 4113 struct mcx_cmdq_entry *cqe; 4114 struct mcx_cmd_set_driver_version_in *in; 4115 struct mcx_cmd_set_driver_version_out *out; 4116 int error; 4117 int token; 4118 uint8_t status; 4119 4120 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4121 token = mcx_cmdq_token(sc); 4122 mcx_cmdq_init(sc, cqe, sizeof(*in) + 4123 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 4124 4125 in = mcx_cmdq_in(cqe); 4126 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 4127 in->cmd_op_mod = htobe16(0); 4128 4129 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4130 &cqe->cq_input_ptr, token) != 0) { 4131 printf(", unable to allocate set driver version mailboxen\n"); 4132 return (-1); 4133 } 4134 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 4135 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 4136 4137 mcx_cmdq_mboxes_sign(&mxm, 1); 4138 mcx_cmdq_post(sc, cqe, 0); 4139 4140 error = mcx_cmdq_poll(sc, cqe, 1000); 4141 if (error != 0) { 4142 printf(", set driver version timeout\n"); 4143 goto free; 4144 } 4145 if (mcx_cmdq_verify(cqe) != 0) { 4146 printf(", set driver version command corrupt\n"); 4147 goto free; 4148 } 4149 4150 status = cqe->cq_output_data[0]; 4151 if (status != MCX_CQ_STATUS_OK) { 4152 printf(", set driver version failed (%x)\n", status); 4153 error = -1; 4154 goto free; 4155 } 4156 4157 free: 4158 mcx_dmamem_free(sc, &mxm); 4159 4160 return (error); 4161 } 4162 4163 static int 4164 mcx_iff(struct mcx_softc *sc) 4165 { 4166 struct ifnet *ifp = &sc->sc_ac.ac_if; 4167 struct mcx_dmamem mxm; 4168 struct mcx_cmdq_entry *cqe; 4169 struct mcx_cmd_modify_nic_vport_context_in *in; 4170 struct mcx_cmd_modify_nic_vport_context_out *out; 4171 struct mcx_nic_vport_ctx *ctx; 4172 int error; 4173 int token; 4174 int insize; 4175 uint32_t dest; 4176 4177 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 4178 sc->sc_rss_flow_table_id; 4179 4180 /* enable or disable the promisc flow */ 4181 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 4182 if (sc->sc_promisc_flow_enabled == 0) { 4183 mcx_set_flow_table_entry_mac(sc, 4184 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 4185 sc->sc_promisc_flow_enabled = 1; 4186 } 4187 } else if (sc->sc_promisc_flow_enabled != 0) { 4188 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 4189 sc->sc_promisc_flow_enabled = 0; 4190 } 4191 4192 /* enable or disable the all-multicast flow */ 4193 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 4194 if (sc->sc_allmulti_flow_enabled == 0) { 4195 uint8_t mcast[ETHER_ADDR_LEN]; 4196 4197 memset(mcast, 0, sizeof(mcast)); 4198 mcast[0] = 0x01; 4199 mcx_set_flow_table_entry_mac(sc, 4200 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 4201 sc->sc_allmulti_flow_enabled = 1; 4202 } 4203 } else if (sc->sc_allmulti_flow_enabled != 0) { 4204 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 4205 sc->sc_allmulti_flow_enabled = 0; 4206 } 4207 4208 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 4209 4210 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4211 token = mcx_cmdq_token(sc); 4212 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4213 4214 in = mcx_cmdq_in(cqe); 4215 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 4216 in->cmd_op_mod = htobe16(0); 4217 in->cmd_field_select = htobe32( 4218 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 4219 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 4220 4221 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4222 printf(", unable to allocate modify " 4223 "nic vport context mailboxen\n"); 4224 return (-1); 4225 } 4226 ctx = (struct mcx_nic_vport_ctx *) 4227 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 4228 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 4229 /* 4230 * always leave promisc-all enabled on the vport since we 4231 * can't give it a vlan list, and we're already doing multicast 4232 * filtering in the flow table. 4233 */ 4234 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 4235 4236 mcx_cmdq_mboxes_sign(&mxm, 1); 4237 mcx_cmdq_post(sc, cqe, 0); 4238 4239 error = mcx_cmdq_poll(sc, cqe, 1000); 4240 if (error != 0) { 4241 printf(", modify nic vport context timeout\n"); 4242 goto free; 4243 } 4244 if (mcx_cmdq_verify(cqe) != 0) { 4245 printf(", modify nic vport context command corrupt\n"); 4246 goto free; 4247 } 4248 4249 out = mcx_cmdq_out(cqe); 4250 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4251 printf(", modify nic vport context failed (%x, %x)\n", 4252 out->cmd_status, betoh32(out->cmd_syndrome)); 4253 error = -1; 4254 goto free; 4255 } 4256 4257 free: 4258 mcx_dmamem_free(sc, &mxm); 4259 4260 return (error); 4261 } 4262 4263 static int 4264 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 4265 { 4266 struct mcx_cmdq_entry *cqe; 4267 struct mcx_cmd_alloc_uar_in *in; 4268 struct mcx_cmd_alloc_uar_out *out; 4269 int error; 4270 4271 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4272 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4273 4274 in = mcx_cmdq_in(cqe); 4275 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 4276 in->cmd_op_mod = htobe16(0); 4277 4278 mcx_cmdq_post(sc, cqe, 0); 4279 4280 error = mcx_cmdq_poll(sc, cqe, 1000); 4281 if (error != 0) { 4282 printf(", alloc uar timeout\n"); 4283 return (-1); 4284 } 4285 if (mcx_cmdq_verify(cqe) != 0) { 4286 printf(", alloc uar command corrupt\n"); 4287 return (-1); 4288 } 4289 4290 out = mcx_cmdq_out(cqe); 4291 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4292 printf(", alloc uar failed (%x)\n", out->cmd_status); 4293 return (-1); 4294 } 4295 4296 *uar = mcx_get_id(out->cmd_uar); 4297 return (0); 4298 } 4299 4300 static int 4301 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4302 uint64_t events, int vector) 4303 { 4304 struct mcx_cmdq_entry *cqe; 4305 struct mcx_dmamem mxm; 4306 struct mcx_cmd_create_eq_in *in; 4307 struct mcx_cmd_create_eq_mb_in *mbin; 4308 struct mcx_cmd_create_eq_out *out; 4309 struct mcx_eq_entry *eqe; 4310 int error; 4311 uint64_t *pas; 4312 int insize, npages, paslen, i, token; 4313 4314 eq->eq_cons = 0; 4315 4316 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4317 MCX_PAGE_SIZE); 4318 paslen = npages * sizeof(*pas); 4319 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4320 4321 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4322 MCX_PAGE_SIZE) != 0) { 4323 printf(", unable to allocate event queue memory\n"); 4324 return (-1); 4325 } 4326 4327 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4328 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4329 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4330 } 4331 4332 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4333 token = mcx_cmdq_token(sc); 4334 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4335 4336 in = mcx_cmdq_in(cqe); 4337 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4338 in->cmd_op_mod = htobe16(0); 4339 4340 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4341 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4342 &cqe->cq_input_ptr, token) != 0) { 4343 printf(", unable to allocate create eq mailboxen\n"); 4344 goto free_eq; 4345 } 4346 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4347 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4348 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4349 mbin->cmd_eq_ctx.eq_intr = vector; 4350 mbin->cmd_event_bitmask = htobe64(events); 4351 4352 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4353 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 4354 4355 /* physical addresses follow the mailbox in data */ 4356 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4357 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4358 mcx_cmdq_post(sc, cqe, 0); 4359 4360 error = mcx_cmdq_poll(sc, cqe, 1000); 4361 if (error != 0) { 4362 printf(", create eq timeout\n"); 4363 goto free_mxm; 4364 } 4365 if (mcx_cmdq_verify(cqe) != 0) { 4366 printf(", create eq command corrupt\n"); 4367 goto free_mxm; 4368 } 4369 4370 out = mcx_cmdq_out(cqe); 4371 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4372 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4373 betoh32(out->cmd_syndrome)); 4374 goto free_mxm; 4375 } 4376 4377 eq->eq_n = mcx_get_id(out->cmd_eqn); 4378 4379 mcx_dmamem_free(sc, &mxm); 4380 4381 mcx_arm_eq(sc, eq, uar); 4382 4383 return (0); 4384 4385 free_mxm: 4386 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4387 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 4388 mcx_dmamem_free(sc, &mxm); 4389 free_eq: 4390 mcx_dmamem_free(sc, &eq->eq_mem); 4391 return (-1); 4392 } 4393 4394 static int 4395 mcx_alloc_pd(struct mcx_softc *sc) 4396 { 4397 struct mcx_cmdq_entry *cqe; 4398 struct mcx_cmd_alloc_pd_in *in; 4399 struct mcx_cmd_alloc_pd_out *out; 4400 int error; 4401 4402 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4403 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4404 4405 in = mcx_cmdq_in(cqe); 4406 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4407 in->cmd_op_mod = htobe16(0); 4408 4409 mcx_cmdq_post(sc, cqe, 0); 4410 4411 error = mcx_cmdq_poll(sc, cqe, 1000); 4412 if (error != 0) { 4413 printf(", alloc pd timeout\n"); 4414 return (-1); 4415 } 4416 if (mcx_cmdq_verify(cqe) != 0) { 4417 printf(", alloc pd command corrupt\n"); 4418 return (-1); 4419 } 4420 4421 out = mcx_cmdq_out(cqe); 4422 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4423 printf(", alloc pd failed (%x)\n", out->cmd_status); 4424 return (-1); 4425 } 4426 4427 sc->sc_pd = mcx_get_id(out->cmd_pd); 4428 return (0); 4429 } 4430 4431 static int 4432 mcx_alloc_tdomain(struct mcx_softc *sc) 4433 { 4434 struct mcx_cmdq_entry *cqe; 4435 struct mcx_cmd_alloc_td_in *in; 4436 struct mcx_cmd_alloc_td_out *out; 4437 int error; 4438 4439 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4440 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4441 4442 in = mcx_cmdq_in(cqe); 4443 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4444 in->cmd_op_mod = htobe16(0); 4445 4446 mcx_cmdq_post(sc, cqe, 0); 4447 4448 error = mcx_cmdq_poll(sc, cqe, 1000); 4449 if (error != 0) { 4450 printf(", alloc transport domain timeout\n"); 4451 return (-1); 4452 } 4453 if (mcx_cmdq_verify(cqe) != 0) { 4454 printf(", alloc transport domain command corrupt\n"); 4455 return (-1); 4456 } 4457 4458 out = mcx_cmdq_out(cqe); 4459 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4460 printf(", alloc transport domain failed (%x)\n", 4461 out->cmd_status); 4462 return (-1); 4463 } 4464 4465 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4466 return (0); 4467 } 4468 4469 static int 4470 mcx_query_nic_vport_context(struct mcx_softc *sc) 4471 { 4472 struct mcx_dmamem mxm; 4473 struct mcx_cmdq_entry *cqe; 4474 struct mcx_cmd_query_nic_vport_context_in *in; 4475 struct mcx_cmd_query_nic_vport_context_out *out; 4476 struct mcx_nic_vport_ctx *ctx; 4477 uint8_t *addr; 4478 int error, token, i; 4479 4480 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4481 token = mcx_cmdq_token(sc); 4482 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4483 4484 in = mcx_cmdq_in(cqe); 4485 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4486 in->cmd_op_mod = htobe16(0); 4487 in->cmd_allowed_list_type = 0; 4488 4489 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4490 &cqe->cq_output_ptr, token) != 0) { 4491 printf(", unable to allocate " 4492 "query nic vport context mailboxen\n"); 4493 return (-1); 4494 } 4495 mcx_cmdq_mboxes_sign(&mxm, 1); 4496 mcx_cmdq_post(sc, cqe, 0); 4497 4498 error = mcx_cmdq_poll(sc, cqe, 1000); 4499 if (error != 0) { 4500 printf(", query nic vport context timeout\n"); 4501 goto free; 4502 } 4503 if (mcx_cmdq_verify(cqe) != 0) { 4504 printf(", query nic vport context command corrupt\n"); 4505 goto free; 4506 } 4507 4508 out = mcx_cmdq_out(cqe); 4509 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4510 printf(", query nic vport context failed (%x, %x)\n", 4511 out->cmd_status, betoh32(out->cmd_syndrome)); 4512 error = -1; 4513 goto free; 4514 } 4515 4516 ctx = (struct mcx_nic_vport_ctx *) 4517 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4518 addr = (uint8_t *)&ctx->vp_perm_addr; 4519 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4520 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 4521 } 4522 free: 4523 mcx_dmamem_free(sc, &mxm); 4524 4525 return (error); 4526 } 4527 4528 static int 4529 mcx_query_special_contexts(struct mcx_softc *sc) 4530 { 4531 struct mcx_cmdq_entry *cqe; 4532 struct mcx_cmd_query_special_ctx_in *in; 4533 struct mcx_cmd_query_special_ctx_out *out; 4534 int error; 4535 4536 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4537 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4538 4539 in = mcx_cmdq_in(cqe); 4540 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4541 in->cmd_op_mod = htobe16(0); 4542 4543 mcx_cmdq_post(sc, cqe, 0); 4544 4545 error = mcx_cmdq_poll(sc, cqe, 1000); 4546 if (error != 0) { 4547 printf(", query special contexts timeout\n"); 4548 return (-1); 4549 } 4550 if (mcx_cmdq_verify(cqe) != 0) { 4551 printf(", query special contexts command corrupt\n"); 4552 return (-1); 4553 } 4554 4555 out = mcx_cmdq_out(cqe); 4556 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4557 printf(", query special contexts failed (%x)\n", 4558 out->cmd_status); 4559 return (-1); 4560 } 4561 4562 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 4563 return (0); 4564 } 4565 4566 static int 4567 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4568 { 4569 struct mcx_reg_pmtu pmtu; 4570 int error; 4571 4572 /* read max mtu */ 4573 memset(&pmtu, 0, sizeof(pmtu)); 4574 pmtu.rp_local_port = 1; 4575 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4576 sizeof(pmtu), MCX_CMDQ_SLOT_POLL); 4577 if (error != 0) { 4578 printf(", unable to get port MTU\n"); 4579 return error; 4580 } 4581 4582 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 4583 pmtu.rp_admin_mtu = htobe16(mtu); 4584 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4585 sizeof(pmtu), MCX_CMDQ_SLOT_POLL); 4586 if (error != 0) { 4587 printf(", unable to set port MTU\n"); 4588 return error; 4589 } 4590 4591 sc->sc_hardmtu = mtu; 4592 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4593 return 0; 4594 } 4595 4596 static int 4597 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4598 { 4599 struct mcx_cmdq_entry *cmde; 4600 struct mcx_cq_entry *cqe; 4601 struct mcx_dmamem mxm; 4602 struct mcx_cmd_create_cq_in *in; 4603 struct mcx_cmd_create_cq_mb_in *mbin; 4604 struct mcx_cmd_create_cq_out *out; 4605 int error; 4606 uint64_t *pas; 4607 int insize, npages, paslen, i, token; 4608 4609 cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4610 4611 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4612 MCX_PAGE_SIZE); 4613 paslen = npages * sizeof(*pas); 4614 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4615 4616 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4617 MCX_PAGE_SIZE) != 0) { 4618 printf("%s: unable to allocate completion queue memory\n", 4619 DEVNAME(sc)); 4620 return (-1); 4621 } 4622 cqe = MCX_DMA_KVA(&cq->cq_mem); 4623 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4624 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4625 } 4626 4627 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4628 token = mcx_cmdq_token(sc); 4629 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4630 4631 in = mcx_cmdq_in(cmde); 4632 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4633 in->cmd_op_mod = htobe16(0); 4634 4635 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4636 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4637 &cmde->cq_input_ptr, token) != 0) { 4638 printf("%s: unable to allocate create cq mailboxen\n", 4639 DEVNAME(sc)); 4640 goto free_cq; 4641 } 4642 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4643 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4644 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4645 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4646 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4647 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4648 MCX_CQ_MOD_COUNTER); 4649 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4650 MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell); 4651 4652 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4653 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 4654 4655 /* physical addresses follow the mailbox in data */ 4656 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4657 mcx_cmdq_post(sc, cmde, 0); 4658 4659 error = mcx_cmdq_poll(sc, cmde, 1000); 4660 if (error != 0) { 4661 printf("%s: create cq timeout\n", DEVNAME(sc)); 4662 goto free_mxm; 4663 } 4664 if (mcx_cmdq_verify(cmde) != 0) { 4665 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4666 goto free_mxm; 4667 } 4668 4669 out = mcx_cmdq_out(cmde); 4670 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4671 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4672 out->cmd_status, betoh32(out->cmd_syndrome)); 4673 goto free_mxm; 4674 } 4675 4676 cq->cq_n = mcx_get_id(out->cmd_cqn); 4677 cq->cq_cons = 0; 4678 cq->cq_count = 0; 4679 4680 mcx_dmamem_free(sc, &mxm); 4681 4682 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4683 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4684 BUS_DMASYNC_PREWRITE); 4685 4686 mcx_arm_cq(sc, cq, uar); 4687 4688 return (0); 4689 4690 free_mxm: 4691 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4692 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4693 mcx_dmamem_free(sc, &mxm); 4694 free_cq: 4695 mcx_dmamem_free(sc, &cq->cq_mem); 4696 return (-1); 4697 } 4698 4699 static int 4700 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4701 { 4702 struct mcx_cmdq_entry *cqe; 4703 struct mcx_cmd_destroy_cq_in *in; 4704 struct mcx_cmd_destroy_cq_out *out; 4705 int error; 4706 int token; 4707 4708 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4709 token = mcx_cmdq_token(sc); 4710 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4711 4712 in = mcx_cmdq_in(cqe); 4713 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4714 in->cmd_op_mod = htobe16(0); 4715 in->cmd_cqn = htobe32(cq->cq_n); 4716 4717 mcx_cmdq_post(sc, cqe, 0); 4718 error = mcx_cmdq_poll(sc, cqe, 1000); 4719 if (error != 0) { 4720 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4721 return error; 4722 } 4723 if (mcx_cmdq_verify(cqe) != 0) { 4724 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4725 return error; 4726 } 4727 4728 out = mcx_cmdq_out(cqe); 4729 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4730 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4731 out->cmd_status, betoh32(out->cmd_syndrome)); 4732 return -1; 4733 } 4734 4735 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4736 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4737 BUS_DMASYNC_POSTWRITE); 4738 4739 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4740 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4741 mcx_dmamem_free(sc, &cq->cq_mem); 4742 4743 cq->cq_n = 0; 4744 cq->cq_cons = 0; 4745 cq->cq_count = 0; 4746 return 0; 4747 } 4748 4749 static int 4750 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4751 { 4752 struct mcx_cmdq_entry *cqe; 4753 struct mcx_dmamem mxm; 4754 struct mcx_cmd_create_rq_in *in; 4755 struct mcx_cmd_create_rq_out *out; 4756 struct mcx_rq_ctx *mbin; 4757 int error; 4758 uint64_t *pas; 4759 uint32_t rq_flags; 4760 int insize, npages, paslen, token; 4761 4762 rx->rx_doorbell = MCX_WQ_DOORBELL_BASE + 4763 (db * MCX_WQ_DOORBELL_STRIDE); 4764 4765 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4766 MCX_PAGE_SIZE); 4767 paslen = npages * sizeof(*pas); 4768 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4769 4770 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4771 MCX_PAGE_SIZE) != 0) { 4772 printf("%s: unable to allocate receive queue memory\n", 4773 DEVNAME(sc)); 4774 return (-1); 4775 } 4776 4777 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4778 token = mcx_cmdq_token(sc); 4779 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4780 4781 in = mcx_cmdq_in(cqe); 4782 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4783 in->cmd_op_mod = htobe16(0); 4784 4785 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4786 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4787 &cqe->cq_input_ptr, token) != 0) { 4788 printf("%s: unable to allocate create rq mailboxen\n", 4789 DEVNAME(sc)); 4790 goto free_rq; 4791 } 4792 mbin = (struct mcx_rq_ctx *) 4793 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4794 rq_flags = MCX_RQ_CTX_RLKEY; 4795 #if NVLAN == 0 4796 rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS; 4797 #endif 4798 mbin->rq_flags = htobe32(rq_flags); 4799 mbin->rq_cqn = htobe32(cqn); 4800 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4801 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4802 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4803 rx->rx_doorbell); 4804 mbin->rq_wq.wq_log_stride = htobe16(4); 4805 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4806 4807 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4808 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 4809 4810 /* physical addresses follow the mailbox in data */ 4811 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4812 mcx_cmdq_post(sc, cqe, 0); 4813 4814 error = mcx_cmdq_poll(sc, cqe, 1000); 4815 if (error != 0) { 4816 printf("%s: create rq timeout\n", DEVNAME(sc)); 4817 goto free_mxm; 4818 } 4819 if (mcx_cmdq_verify(cqe) != 0) { 4820 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4821 goto free_mxm; 4822 } 4823 4824 out = mcx_cmdq_out(cqe); 4825 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4826 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4827 out->cmd_status, betoh32(out->cmd_syndrome)); 4828 goto free_mxm; 4829 } 4830 4831 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4832 4833 mcx_dmamem_free(sc, &mxm); 4834 4835 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4836 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 4837 4838 return (0); 4839 4840 free_mxm: 4841 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4842 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4843 mcx_dmamem_free(sc, &mxm); 4844 free_rq: 4845 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4846 return (-1); 4847 } 4848 4849 static int 4850 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4851 { 4852 struct mcx_cmdq_entry *cqe; 4853 struct mcx_dmamem mxm; 4854 struct mcx_cmd_modify_rq_in *in; 4855 struct mcx_cmd_modify_rq_mb_in *mbin; 4856 struct mcx_cmd_modify_rq_out *out; 4857 int error; 4858 int token; 4859 4860 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4861 token = mcx_cmdq_token(sc); 4862 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4863 sizeof(*out), token); 4864 4865 in = mcx_cmdq_in(cqe); 4866 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4867 in->cmd_op_mod = htobe16(0); 4868 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4869 4870 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4871 &cqe->cq_input_ptr, token) != 0) { 4872 printf("%s: unable to allocate modify rq mailbox\n", 4873 DEVNAME(sc)); 4874 return (-1); 4875 } 4876 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4877 mbin->cmd_rq_ctx.rq_flags = htobe32( 4878 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4879 4880 mcx_cmdq_mboxes_sign(&mxm, 1); 4881 mcx_cmdq_post(sc, cqe, 0); 4882 error = mcx_cmdq_poll(sc, cqe, 1000); 4883 if (error != 0) { 4884 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4885 goto free; 4886 } 4887 if (mcx_cmdq_verify(cqe) != 0) { 4888 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4889 goto free; 4890 } 4891 4892 out = mcx_cmdq_out(cqe); 4893 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4894 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4895 out->cmd_status, betoh32(out->cmd_syndrome)); 4896 error = -1; 4897 goto free; 4898 } 4899 4900 free: 4901 mcx_dmamem_free(sc, &mxm); 4902 return (error); 4903 } 4904 4905 static int 4906 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4907 { 4908 struct mcx_cmdq_entry *cqe; 4909 struct mcx_cmd_destroy_rq_in *in; 4910 struct mcx_cmd_destroy_rq_out *out; 4911 int error; 4912 int token; 4913 4914 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4915 token = mcx_cmdq_token(sc); 4916 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4917 4918 in = mcx_cmdq_in(cqe); 4919 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4920 in->cmd_op_mod = htobe16(0); 4921 in->cmd_rqn = htobe32(rx->rx_rqn); 4922 4923 mcx_cmdq_post(sc, cqe, 0); 4924 error = mcx_cmdq_poll(sc, cqe, 1000); 4925 if (error != 0) { 4926 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4927 return error; 4928 } 4929 if (mcx_cmdq_verify(cqe) != 0) { 4930 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4931 return error; 4932 } 4933 4934 out = mcx_cmdq_out(cqe); 4935 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4936 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4937 out->cmd_status, betoh32(out->cmd_syndrome)); 4938 return -1; 4939 } 4940 4941 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4942 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 4943 4944 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4945 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4946 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4947 4948 rx->rx_rqn = 0; 4949 return 0; 4950 } 4951 4952 static int 4953 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 4954 { 4955 struct mcx_cmdq_entry *cqe; 4956 struct mcx_dmamem mxm; 4957 struct mcx_cmd_create_tir_in *in; 4958 struct mcx_cmd_create_tir_mb_in *mbin; 4959 struct mcx_cmd_create_tir_out *out; 4960 int error; 4961 int token; 4962 4963 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4964 token = mcx_cmdq_token(sc); 4965 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4966 sizeof(*out), token); 4967 4968 in = mcx_cmdq_in(cqe); 4969 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4970 in->cmd_op_mod = htobe16(0); 4971 4972 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4973 &cqe->cq_input_ptr, token) != 0) { 4974 printf("%s: unable to allocate create tir mailbox\n", 4975 DEVNAME(sc)); 4976 return (-1); 4977 } 4978 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4979 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4980 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 4981 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4982 4983 mcx_cmdq_post(sc, cqe, 0); 4984 error = mcx_cmdq_poll(sc, cqe, 1000); 4985 if (error != 0) { 4986 printf("%s: create tir timeout\n", DEVNAME(sc)); 4987 goto free; 4988 } 4989 if (mcx_cmdq_verify(cqe) != 0) { 4990 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4991 goto free; 4992 } 4993 4994 out = mcx_cmdq_out(cqe); 4995 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4996 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4997 out->cmd_status, betoh32(out->cmd_syndrome)); 4998 error = -1; 4999 goto free; 5000 } 5001 5002 *tirn = mcx_get_id(out->cmd_tirn); 5003 free: 5004 mcx_dmamem_free(sc, &mxm); 5005 return (error); 5006 } 5007 5008 static int 5009 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 5010 int *tirn) 5011 { 5012 struct mcx_cmdq_entry *cqe; 5013 struct mcx_dmamem mxm; 5014 struct mcx_cmd_create_tir_in *in; 5015 struct mcx_cmd_create_tir_mb_in *mbin; 5016 struct mcx_cmd_create_tir_out *out; 5017 int error; 5018 int token; 5019 5020 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5021 token = mcx_cmdq_token(sc); 5022 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5023 sizeof(*out), token); 5024 5025 in = mcx_cmdq_in(cqe); 5026 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 5027 in->cmd_op_mod = htobe16(0); 5028 5029 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5030 &cqe->cq_input_ptr, token) != 0) { 5031 printf("%s: unable to allocate create tir mailbox\n", 5032 DEVNAME(sc)); 5033 return (-1); 5034 } 5035 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5036 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 5037 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 5038 mbin->cmd_indir_table = htobe32(rqtn); 5039 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 5040 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 5041 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 5042 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 5043 sizeof(mbin->cmd_rx_hash_key)); 5044 5045 mcx_cmdq_post(sc, cqe, 0); 5046 error = mcx_cmdq_poll(sc, cqe, 1000); 5047 if (error != 0) { 5048 printf("%s: create tir timeout\n", DEVNAME(sc)); 5049 goto free; 5050 } 5051 if (mcx_cmdq_verify(cqe) != 0) { 5052 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 5053 goto free; 5054 } 5055 5056 out = mcx_cmdq_out(cqe); 5057 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5058 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 5059 out->cmd_status, betoh32(out->cmd_syndrome)); 5060 error = -1; 5061 goto free; 5062 } 5063 5064 *tirn = mcx_get_id(out->cmd_tirn); 5065 free: 5066 mcx_dmamem_free(sc, &mxm); 5067 return (error); 5068 } 5069 5070 static int 5071 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 5072 { 5073 struct mcx_cmdq_entry *cqe; 5074 struct mcx_cmd_destroy_tir_in *in; 5075 struct mcx_cmd_destroy_tir_out *out; 5076 int error; 5077 int token; 5078 5079 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5080 token = mcx_cmdq_token(sc); 5081 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5082 5083 in = mcx_cmdq_in(cqe); 5084 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 5085 in->cmd_op_mod = htobe16(0); 5086 in->cmd_tirn = htobe32(tirn); 5087 5088 mcx_cmdq_post(sc, cqe, 0); 5089 error = mcx_cmdq_poll(sc, cqe, 1000); 5090 if (error != 0) { 5091 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 5092 return error; 5093 } 5094 if (mcx_cmdq_verify(cqe) != 0) { 5095 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 5096 return error; 5097 } 5098 5099 out = mcx_cmdq_out(cqe); 5100 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5101 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 5102 out->cmd_status, betoh32(out->cmd_syndrome)); 5103 return -1; 5104 } 5105 5106 return (0); 5107 } 5108 5109 static int 5110 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 5111 int cqn) 5112 { 5113 struct mcx_cmdq_entry *cqe; 5114 struct mcx_dmamem mxm; 5115 struct mcx_cmd_create_sq_in *in; 5116 struct mcx_sq_ctx *mbin; 5117 struct mcx_cmd_create_sq_out *out; 5118 int error; 5119 uint64_t *pas; 5120 int insize, npages, paslen, token; 5121 5122 tx->tx_doorbell = MCX_WQ_DOORBELL_BASE + 5123 (db * MCX_WQ_DOORBELL_STRIDE) + 4; 5124 5125 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 5126 MCX_PAGE_SIZE); 5127 paslen = npages * sizeof(*pas); 5128 insize = sizeof(struct mcx_sq_ctx) + paslen; 5129 5130 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 5131 MCX_PAGE_SIZE) != 0) { 5132 printf("%s: unable to allocate send queue memory\n", 5133 DEVNAME(sc)); 5134 return (-1); 5135 } 5136 5137 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5138 token = mcx_cmdq_token(sc); 5139 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 5140 token); 5141 5142 in = mcx_cmdq_in(cqe); 5143 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 5144 in->cmd_op_mod = htobe16(0); 5145 5146 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 5147 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 5148 &cqe->cq_input_ptr, token) != 0) { 5149 printf("%s: unable to allocate create sq mailboxen\n", 5150 DEVNAME(sc)); 5151 goto free_sq; 5152 } 5153 mbin = (struct mcx_sq_ctx *) 5154 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 5155 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 5156 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 5157 mbin->sq_cqn = htobe32(cqn); 5158 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 5159 mbin->sq_tis_num = htobe32(sc->sc_tis); 5160 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 5161 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 5162 mbin->sq_wq.wq_uar_page = htobe32(uar); 5163 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 5164 tx->tx_doorbell); 5165 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 5166 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 5167 5168 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5169 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 5170 5171 /* physical addresses follow the mailbox in data */ 5172 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 5173 npages, &tx->tx_sq_mem); 5174 mcx_cmdq_post(sc, cqe, 0); 5175 5176 error = mcx_cmdq_poll(sc, cqe, 1000); 5177 if (error != 0) { 5178 printf("%s: create sq timeout\n", DEVNAME(sc)); 5179 goto free_mxm; 5180 } 5181 if (mcx_cmdq_verify(cqe) != 0) { 5182 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 5183 goto free_mxm; 5184 } 5185 5186 out = mcx_cmdq_out(cqe); 5187 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5188 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 5189 out->cmd_status, betoh32(out->cmd_syndrome)); 5190 goto free_mxm; 5191 } 5192 5193 tx->tx_uar = uar; 5194 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 5195 5196 mcx_dmamem_free(sc, &mxm); 5197 5198 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5199 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 5200 5201 return (0); 5202 5203 free_mxm: 5204 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5205 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5206 mcx_dmamem_free(sc, &mxm); 5207 free_sq: 5208 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5209 return (-1); 5210 } 5211 5212 static int 5213 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5214 { 5215 struct mcx_cmdq_entry *cqe; 5216 struct mcx_cmd_destroy_sq_in *in; 5217 struct mcx_cmd_destroy_sq_out *out; 5218 int error; 5219 int token; 5220 5221 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5222 token = mcx_cmdq_token(sc); 5223 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5224 5225 in = mcx_cmdq_in(cqe); 5226 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 5227 in->cmd_op_mod = htobe16(0); 5228 in->cmd_sqn = htobe32(tx->tx_sqn); 5229 5230 mcx_cmdq_post(sc, cqe, 0); 5231 error = mcx_cmdq_poll(sc, cqe, 1000); 5232 if (error != 0) { 5233 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 5234 return error; 5235 } 5236 if (mcx_cmdq_verify(cqe) != 0) { 5237 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 5238 return error; 5239 } 5240 5241 out = mcx_cmdq_out(cqe); 5242 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5243 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 5244 out->cmd_status, betoh32(out->cmd_syndrome)); 5245 return -1; 5246 } 5247 5248 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5249 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 5250 5251 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5252 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5253 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5254 5255 tx->tx_sqn = 0; 5256 return 0; 5257 } 5258 5259 static int 5260 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5261 { 5262 struct mcx_cmdq_entry *cqe; 5263 struct mcx_dmamem mxm; 5264 struct mcx_cmd_modify_sq_in *in; 5265 struct mcx_cmd_modify_sq_mb_in *mbin; 5266 struct mcx_cmd_modify_sq_out *out; 5267 int error; 5268 int token; 5269 5270 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5271 token = mcx_cmdq_token(sc); 5272 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5273 sizeof(*out), token); 5274 5275 in = mcx_cmdq_in(cqe); 5276 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 5277 in->cmd_op_mod = htobe16(0); 5278 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 5279 5280 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5281 &cqe->cq_input_ptr, token) != 0) { 5282 printf("%s: unable to allocate modify sq mailbox\n", 5283 DEVNAME(sc)); 5284 return (-1); 5285 } 5286 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5287 mbin->cmd_sq_ctx.sq_flags = htobe32( 5288 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 5289 5290 mcx_cmdq_mboxes_sign(&mxm, 1); 5291 mcx_cmdq_post(sc, cqe, 0); 5292 error = mcx_cmdq_poll(sc, cqe, 1000); 5293 if (error != 0) { 5294 printf("%s: modify sq timeout\n", DEVNAME(sc)); 5295 goto free; 5296 } 5297 if (mcx_cmdq_verify(cqe) != 0) { 5298 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 5299 goto free; 5300 } 5301 5302 out = mcx_cmdq_out(cqe); 5303 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5304 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 5305 out->cmd_status, betoh32(out->cmd_syndrome)); 5306 error = -1; 5307 goto free; 5308 } 5309 5310 free: 5311 mcx_dmamem_free(sc, &mxm); 5312 return (error); 5313 } 5314 5315 static int 5316 mcx_create_tis(struct mcx_softc *sc, int *tis) 5317 { 5318 struct mcx_cmdq_entry *cqe; 5319 struct mcx_dmamem mxm; 5320 struct mcx_cmd_create_tis_in *in; 5321 struct mcx_cmd_create_tis_mb_in *mbin; 5322 struct mcx_cmd_create_tis_out *out; 5323 int error; 5324 int token; 5325 5326 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5327 token = mcx_cmdq_token(sc); 5328 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5329 sizeof(*out), token); 5330 5331 in = mcx_cmdq_in(cqe); 5332 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 5333 in->cmd_op_mod = htobe16(0); 5334 5335 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5336 &cqe->cq_input_ptr, token) != 0) { 5337 printf("%s: unable to allocate create tis mailbox\n", 5338 DEVNAME(sc)); 5339 return (-1); 5340 } 5341 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5342 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5343 5344 mcx_cmdq_mboxes_sign(&mxm, 1); 5345 mcx_cmdq_post(sc, cqe, 0); 5346 error = mcx_cmdq_poll(sc, cqe, 1000); 5347 if (error != 0) { 5348 printf("%s: create tis timeout\n", DEVNAME(sc)); 5349 goto free; 5350 } 5351 if (mcx_cmdq_verify(cqe) != 0) { 5352 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5353 goto free; 5354 } 5355 5356 out = mcx_cmdq_out(cqe); 5357 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5358 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5359 out->cmd_status, betoh32(out->cmd_syndrome)); 5360 error = -1; 5361 goto free; 5362 } 5363 5364 *tis = mcx_get_id(out->cmd_tisn); 5365 free: 5366 mcx_dmamem_free(sc, &mxm); 5367 return (error); 5368 } 5369 5370 static int 5371 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5372 { 5373 struct mcx_cmdq_entry *cqe; 5374 struct mcx_cmd_destroy_tis_in *in; 5375 struct mcx_cmd_destroy_tis_out *out; 5376 int error; 5377 int token; 5378 5379 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5380 token = mcx_cmdq_token(sc); 5381 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5382 5383 in = mcx_cmdq_in(cqe); 5384 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5385 in->cmd_op_mod = htobe16(0); 5386 in->cmd_tisn = htobe32(tis); 5387 5388 mcx_cmdq_post(sc, cqe, 0); 5389 error = mcx_cmdq_poll(sc, cqe, 1000); 5390 if (error != 0) { 5391 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5392 return error; 5393 } 5394 if (mcx_cmdq_verify(cqe) != 0) { 5395 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5396 return error; 5397 } 5398 5399 out = mcx_cmdq_out(cqe); 5400 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5401 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5402 out->cmd_status, betoh32(out->cmd_syndrome)); 5403 return -1; 5404 } 5405 5406 return 0; 5407 } 5408 5409 static int 5410 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5411 { 5412 struct mcx_cmdq_entry *cqe; 5413 struct mcx_dmamem mxm; 5414 struct mcx_cmd_create_rqt_in *in; 5415 struct mcx_cmd_create_rqt_mb_in *mbin; 5416 struct mcx_cmd_create_rqt_out *out; 5417 struct mcx_rqt_ctx *rqt_ctx; 5418 int *rqtn; 5419 int error; 5420 int token; 5421 int i; 5422 5423 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5424 token = mcx_cmdq_token(sc); 5425 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5426 (size * sizeof(int)), sizeof(*out), token); 5427 5428 in = mcx_cmdq_in(cqe); 5429 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5430 in->cmd_op_mod = htobe16(0); 5431 5432 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5433 &cqe->cq_input_ptr, token) != 0) { 5434 printf("%s: unable to allocate create rqt mailbox\n", 5435 DEVNAME(sc)); 5436 return (-1); 5437 } 5438 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5439 rqt_ctx = &mbin->cmd_rqt; 5440 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5441 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5442 5443 /* rqt list follows the rqt context */ 5444 rqtn = (int *)(rqt_ctx + 1); 5445 for (i = 0; i < size; i++) { 5446 rqtn[i] = htobe32(rqns[i]); 5447 } 5448 5449 mcx_cmdq_mboxes_sign(&mxm, 1); 5450 mcx_cmdq_post(sc, cqe, 0); 5451 error = mcx_cmdq_poll(sc, cqe, 1000); 5452 if (error != 0) { 5453 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5454 goto free; 5455 } 5456 if (mcx_cmdq_verify(cqe) != 0) { 5457 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5458 goto free; 5459 } 5460 5461 out = mcx_cmdq_out(cqe); 5462 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5463 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5464 out->cmd_status, betoh32(out->cmd_syndrome)); 5465 error = -1; 5466 goto free; 5467 } 5468 5469 *rqt = mcx_get_id(out->cmd_rqtn); 5470 return (0); 5471 free: 5472 mcx_dmamem_free(sc, &mxm); 5473 return (error); 5474 } 5475 5476 static int 5477 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5478 { 5479 struct mcx_cmdq_entry *cqe; 5480 struct mcx_cmd_destroy_rqt_in *in; 5481 struct mcx_cmd_destroy_rqt_out *out; 5482 int error; 5483 int token; 5484 5485 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5486 token = mcx_cmdq_token(sc); 5487 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5488 5489 in = mcx_cmdq_in(cqe); 5490 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5491 in->cmd_op_mod = htobe16(0); 5492 in->cmd_rqtn = htobe32(rqt); 5493 5494 mcx_cmdq_post(sc, cqe, 0); 5495 error = mcx_cmdq_poll(sc, cqe, 1000); 5496 if (error != 0) { 5497 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5498 return error; 5499 } 5500 if (mcx_cmdq_verify(cqe) != 0) { 5501 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5502 return error; 5503 } 5504 5505 out = mcx_cmdq_out(cqe); 5506 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5507 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5508 out->cmd_status, betoh32(out->cmd_syndrome)); 5509 return -1; 5510 } 5511 5512 return 0; 5513 } 5514 5515 #if 0 5516 static int 5517 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5518 { 5519 struct mcx_cmdq_entry *cqe; 5520 struct mcx_cmd_alloc_flow_counter_in *in; 5521 struct mcx_cmd_alloc_flow_counter_out *out; 5522 int error; 5523 5524 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5525 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5526 5527 in = mcx_cmdq_in(cqe); 5528 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5529 in->cmd_op_mod = htobe16(0); 5530 5531 mcx_cmdq_post(sc, cqe, 0); 5532 5533 error = mcx_cmdq_poll(sc, cqe, 1000); 5534 if (error != 0) { 5535 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5536 return (-1); 5537 } 5538 if (mcx_cmdq_verify(cqe) != 0) { 5539 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5540 return (-1); 5541 } 5542 5543 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5544 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5545 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5546 out->cmd_status); 5547 return (-1); 5548 } 5549 5550 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 5551 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5552 5553 return (0); 5554 } 5555 #endif 5556 5557 static int 5558 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5559 int *flow_table_id) 5560 { 5561 struct mcx_cmdq_entry *cqe; 5562 struct mcx_dmamem mxm; 5563 struct mcx_cmd_create_flow_table_in *in; 5564 struct mcx_cmd_create_flow_table_mb_in *mbin; 5565 struct mcx_cmd_create_flow_table_out *out; 5566 int error; 5567 int token; 5568 5569 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5570 token = mcx_cmdq_token(sc); 5571 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5572 sizeof(*out), token); 5573 5574 in = mcx_cmdq_in(cqe); 5575 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5576 in->cmd_op_mod = htobe16(0); 5577 5578 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5579 &cqe->cq_input_ptr, token) != 0) { 5580 printf("%s: unable to allocate create flow table mailbox\n", 5581 DEVNAME(sc)); 5582 return (-1); 5583 } 5584 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5585 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5586 mbin->cmd_ctx.ft_log_size = log_size; 5587 mbin->cmd_ctx.ft_level = level; 5588 5589 mcx_cmdq_mboxes_sign(&mxm, 1); 5590 mcx_cmdq_post(sc, cqe, 0); 5591 error = mcx_cmdq_poll(sc, cqe, 1000); 5592 if (error != 0) { 5593 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5594 goto free; 5595 } 5596 if (mcx_cmdq_verify(cqe) != 0) { 5597 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5598 goto free; 5599 } 5600 5601 out = mcx_cmdq_out(cqe); 5602 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5603 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5604 out->cmd_status, betoh32(out->cmd_syndrome)); 5605 error = -1; 5606 goto free; 5607 } 5608 5609 *flow_table_id = mcx_get_id(out->cmd_table_id); 5610 free: 5611 mcx_dmamem_free(sc, &mxm); 5612 return (error); 5613 } 5614 5615 static int 5616 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5617 { 5618 struct mcx_cmdq_entry *cqe; 5619 struct mcx_dmamem mxm; 5620 struct mcx_cmd_set_flow_table_root_in *in; 5621 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5622 struct mcx_cmd_set_flow_table_root_out *out; 5623 int error; 5624 int token; 5625 5626 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5627 token = mcx_cmdq_token(sc); 5628 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5629 sizeof(*out), token); 5630 5631 in = mcx_cmdq_in(cqe); 5632 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5633 in->cmd_op_mod = htobe16(0); 5634 5635 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5636 &cqe->cq_input_ptr, token) != 0) { 5637 printf("%s: unable to allocate set flow table root mailbox\n", 5638 DEVNAME(sc)); 5639 return (-1); 5640 } 5641 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5642 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5643 mbin->cmd_table_id = htobe32(flow_table_id); 5644 5645 mcx_cmdq_mboxes_sign(&mxm, 1); 5646 mcx_cmdq_post(sc, cqe, 0); 5647 error = mcx_cmdq_poll(sc, cqe, 1000); 5648 if (error != 0) { 5649 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5650 goto free; 5651 } 5652 if (mcx_cmdq_verify(cqe) != 0) { 5653 printf("%s: set flow table root command corrupt\n", 5654 DEVNAME(sc)); 5655 goto free; 5656 } 5657 5658 out = mcx_cmdq_out(cqe); 5659 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5660 printf("%s: set flow table root failed (%x, %x)\n", 5661 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5662 error = -1; 5663 goto free; 5664 } 5665 5666 free: 5667 mcx_dmamem_free(sc, &mxm); 5668 return (error); 5669 } 5670 5671 static int 5672 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5673 { 5674 struct mcx_cmdq_entry *cqe; 5675 struct mcx_dmamem mxm; 5676 struct mcx_cmd_destroy_flow_table_in *in; 5677 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5678 struct mcx_cmd_destroy_flow_table_out *out; 5679 int error; 5680 int token; 5681 5682 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5683 token = mcx_cmdq_token(sc); 5684 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5685 5686 in = mcx_cmdq_in(cqe); 5687 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5688 in->cmd_op_mod = htobe16(0); 5689 5690 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5691 &cqe->cq_input_ptr, token) != 0) { 5692 printf("%s: unable to allocate destroy flow table mailbox\n", 5693 DEVNAME(sc)); 5694 return (-1); 5695 } 5696 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5697 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5698 mb->cmd_table_id = htobe32(flow_table_id); 5699 5700 mcx_cmdq_mboxes_sign(&mxm, 1); 5701 mcx_cmdq_post(sc, cqe, 0); 5702 error = mcx_cmdq_poll(sc, cqe, 1000); 5703 if (error != 0) { 5704 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5705 goto free; 5706 } 5707 if (mcx_cmdq_verify(cqe) != 0) { 5708 printf("%s: destroy flow table command corrupt\n", 5709 DEVNAME(sc)); 5710 goto free; 5711 } 5712 5713 out = mcx_cmdq_out(cqe); 5714 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5715 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5716 out->cmd_status, betoh32(out->cmd_syndrome)); 5717 error = -1; 5718 goto free; 5719 } 5720 5721 free: 5722 mcx_dmamem_free(sc, &mxm); 5723 return (error); 5724 } 5725 5726 5727 static int 5728 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5729 int start, int size, int match_enable, struct mcx_flow_match *match) 5730 { 5731 struct mcx_cmdq_entry *cqe; 5732 struct mcx_dmamem mxm; 5733 struct mcx_cmd_create_flow_group_in *in; 5734 struct mcx_cmd_create_flow_group_mb_in *mbin; 5735 struct mcx_cmd_create_flow_group_out *out; 5736 struct mcx_flow_group *mfg; 5737 int error; 5738 int token; 5739 5740 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5741 token = mcx_cmdq_token(sc); 5742 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5743 token); 5744 5745 in = mcx_cmdq_in(cqe); 5746 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5747 in->cmd_op_mod = htobe16(0); 5748 5749 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5750 != 0) { 5751 printf("%s: unable to allocate create flow group mailbox\n", 5752 DEVNAME(sc)); 5753 return (-1); 5754 } 5755 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5756 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5757 mbin->cmd_table_id = htobe32(flow_table_id); 5758 mbin->cmd_start_flow_index = htobe32(start); 5759 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5760 5761 mbin->cmd_match_criteria_enable = match_enable; 5762 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5763 5764 mcx_cmdq_mboxes_sign(&mxm, 2); 5765 mcx_cmdq_post(sc, cqe, 0); 5766 error = mcx_cmdq_poll(sc, cqe, 1000); 5767 if (error != 0) { 5768 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5769 goto free; 5770 } 5771 if (mcx_cmdq_verify(cqe) != 0) { 5772 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5773 goto free; 5774 } 5775 5776 out = mcx_cmdq_out(cqe); 5777 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5778 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5779 out->cmd_status, betoh32(out->cmd_syndrome)); 5780 error = -1; 5781 goto free; 5782 } 5783 5784 mfg = &sc->sc_flow_group[group]; 5785 mfg->g_id = mcx_get_id(out->cmd_group_id); 5786 mfg->g_table = flow_table_id; 5787 mfg->g_start = start; 5788 mfg->g_size = size; 5789 5790 free: 5791 mcx_dmamem_free(sc, &mxm); 5792 return (error); 5793 } 5794 5795 static int 5796 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5797 { 5798 struct mcx_cmdq_entry *cqe; 5799 struct mcx_dmamem mxm; 5800 struct mcx_cmd_destroy_flow_group_in *in; 5801 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5802 struct mcx_cmd_destroy_flow_group_out *out; 5803 struct mcx_flow_group *mfg; 5804 int error; 5805 int token; 5806 5807 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5808 token = mcx_cmdq_token(sc); 5809 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5810 5811 in = mcx_cmdq_in(cqe); 5812 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5813 in->cmd_op_mod = htobe16(0); 5814 5815 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5816 &cqe->cq_input_ptr, token) != 0) { 5817 printf("%s: unable to allocate destroy flow group mailbox\n", 5818 DEVNAME(sc)); 5819 return (-1); 5820 } 5821 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5822 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5823 mfg = &sc->sc_flow_group[group]; 5824 mb->cmd_table_id = htobe32(mfg->g_table); 5825 mb->cmd_group_id = htobe32(mfg->g_id); 5826 5827 mcx_cmdq_mboxes_sign(&mxm, 2); 5828 mcx_cmdq_post(sc, cqe, 0); 5829 error = mcx_cmdq_poll(sc, cqe, 1000); 5830 if (error != 0) { 5831 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5832 goto free; 5833 } 5834 if (mcx_cmdq_verify(cqe) != 0) { 5835 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5836 goto free; 5837 } 5838 5839 out = mcx_cmdq_out(cqe); 5840 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5841 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5842 out->cmd_status, betoh32(out->cmd_syndrome)); 5843 error = -1; 5844 goto free; 5845 } 5846 5847 mfg->g_id = -1; 5848 mfg->g_table = -1; 5849 mfg->g_size = 0; 5850 mfg->g_start = 0; 5851 free: 5852 mcx_dmamem_free(sc, &mxm); 5853 return (error); 5854 } 5855 5856 static int 5857 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5858 uint8_t *macaddr, uint32_t dest) 5859 { 5860 struct mcx_cmdq_entry *cqe; 5861 struct mcx_dmamem mxm; 5862 struct mcx_cmd_set_flow_table_entry_in *in; 5863 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5864 struct mcx_cmd_set_flow_table_entry_out *out; 5865 struct mcx_flow_group *mfg; 5866 uint32_t *pdest; 5867 int error; 5868 int token; 5869 5870 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5871 token = mcx_cmdq_token(sc); 5872 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5873 sizeof(*out), token); 5874 5875 in = mcx_cmdq_in(cqe); 5876 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5877 in->cmd_op_mod = htobe16(0); 5878 5879 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5880 != 0) { 5881 printf("%s: unable to allocate set flow table entry mailbox\n", 5882 DEVNAME(sc)); 5883 return (-1); 5884 } 5885 5886 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5887 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5888 5889 mfg = &sc->sc_flow_group[group]; 5890 mbin->cmd_table_id = htobe32(mfg->g_table); 5891 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5892 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5893 5894 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5895 pdest = (uint32_t *) 5896 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5897 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5898 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5899 *pdest = htobe32(dest); 5900 5901 /* the only thing we match on at the moment is the dest mac address */ 5902 if (macaddr != NULL) { 5903 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5904 ETHER_ADDR_LEN); 5905 } 5906 5907 mcx_cmdq_mboxes_sign(&mxm, 2); 5908 mcx_cmdq_post(sc, cqe, 0); 5909 error = mcx_cmdq_poll(sc, cqe, 1000); 5910 if (error != 0) { 5911 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5912 goto free; 5913 } 5914 if (mcx_cmdq_verify(cqe) != 0) { 5915 printf("%s: set flow table entry command corrupt\n", 5916 DEVNAME(sc)); 5917 goto free; 5918 } 5919 5920 out = mcx_cmdq_out(cqe); 5921 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5922 printf("%s: set flow table entry failed (%x, %x)\n", 5923 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5924 error = -1; 5925 goto free; 5926 } 5927 5928 free: 5929 mcx_dmamem_free(sc, &mxm); 5930 return (error); 5931 } 5932 5933 static int 5934 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5935 int ethertype, int ip_proto, uint32_t dest) 5936 { 5937 struct mcx_cmdq_entry *cqe; 5938 struct mcx_dmamem mxm; 5939 struct mcx_cmd_set_flow_table_entry_in *in; 5940 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5941 struct mcx_cmd_set_flow_table_entry_out *out; 5942 struct mcx_flow_group *mfg; 5943 uint32_t *pdest; 5944 int error; 5945 int token; 5946 5947 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5948 token = mcx_cmdq_token(sc); 5949 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5950 sizeof(*out), token); 5951 5952 in = mcx_cmdq_in(cqe); 5953 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5954 in->cmd_op_mod = htobe16(0); 5955 5956 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5957 != 0) { 5958 printf("%s: unable to allocate set flow table entry mailbox\n", 5959 DEVNAME(sc)); 5960 return (-1); 5961 } 5962 5963 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5964 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5965 5966 mfg = &sc->sc_flow_group[group]; 5967 mbin->cmd_table_id = htobe32(mfg->g_table); 5968 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5969 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5970 5971 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5972 pdest = (uint32_t *) 5973 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5974 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5975 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5976 *pdest = htobe32(dest); 5977 5978 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 5979 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 5980 5981 mcx_cmdq_mboxes_sign(&mxm, 2); 5982 mcx_cmdq_post(sc, cqe, 0); 5983 error = mcx_cmdq_poll(sc, cqe, 1000); 5984 if (error != 0) { 5985 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5986 goto free; 5987 } 5988 if (mcx_cmdq_verify(cqe) != 0) { 5989 printf("%s: set flow table entry command corrupt\n", 5990 DEVNAME(sc)); 5991 goto free; 5992 } 5993 5994 out = mcx_cmdq_out(cqe); 5995 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5996 printf("%s: set flow table entry failed (%x, %x)\n", 5997 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5998 error = -1; 5999 goto free; 6000 } 6001 6002 free: 6003 mcx_dmamem_free(sc, &mxm); 6004 return (error); 6005 } 6006 6007 static int 6008 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 6009 { 6010 struct mcx_cmdq_entry *cqe; 6011 struct mcx_dmamem mxm; 6012 struct mcx_cmd_delete_flow_table_entry_in *in; 6013 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 6014 struct mcx_cmd_delete_flow_table_entry_out *out; 6015 struct mcx_flow_group *mfg; 6016 int error; 6017 int token; 6018 6019 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6020 token = mcx_cmdq_token(sc); 6021 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 6022 token); 6023 6024 in = mcx_cmdq_in(cqe); 6025 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 6026 in->cmd_op_mod = htobe16(0); 6027 6028 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6029 &cqe->cq_input_ptr, token) != 0) { 6030 printf("%s: unable to allocate " 6031 "delete flow table entry mailbox\n", DEVNAME(sc)); 6032 return (-1); 6033 } 6034 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6035 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 6036 6037 mfg = &sc->sc_flow_group[group]; 6038 mbin->cmd_table_id = htobe32(mfg->g_table); 6039 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 6040 6041 mcx_cmdq_mboxes_sign(&mxm, 2); 6042 mcx_cmdq_post(sc, cqe, 0); 6043 error = mcx_cmdq_poll(sc, cqe, 1000); 6044 if (error != 0) { 6045 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 6046 goto free; 6047 } 6048 if (mcx_cmdq_verify(cqe) != 0) { 6049 printf("%s: delete flow table entry command corrupt\n", 6050 DEVNAME(sc)); 6051 goto free; 6052 } 6053 6054 out = mcx_cmdq_out(cqe); 6055 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6056 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 6057 DEVNAME(sc), group, index, out->cmd_status, 6058 betoh32(out->cmd_syndrome)); 6059 error = -1; 6060 goto free; 6061 } 6062 6063 free: 6064 mcx_dmamem_free(sc, &mxm); 6065 return (error); 6066 } 6067 6068 #if 0 6069 int 6070 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 6071 { 6072 struct mcx_dmamem mxm; 6073 struct mcx_cmdq_entry *cqe; 6074 struct mcx_cmd_query_flow_table_in *in; 6075 struct mcx_cmd_query_flow_table_mb_in *mbin; 6076 struct mcx_cmd_query_flow_table_out *out; 6077 struct mcx_cmd_query_flow_table_mb_out *mbout; 6078 uint8_t token = mcx_cmdq_token(sc); 6079 int error; 6080 int i; 6081 uint8_t *dump; 6082 6083 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6084 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6085 sizeof(*out) + sizeof(*mbout) + 16, token); 6086 6087 in = mcx_cmdq_in(cqe); 6088 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 6089 in->cmd_op_mod = htobe16(0); 6090 6091 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6092 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 6093 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6094 &cqe->cq_output_ptr, token) != 0) { 6095 printf(", unable to allocate query flow table mailboxes\n"); 6096 return (-1); 6097 } 6098 cqe->cq_input_ptr = cqe->cq_output_ptr; 6099 6100 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6101 mbin->cmd_table_type = 0; 6102 mbin->cmd_table_id = htobe32(flow_table_id); 6103 6104 mcx_cmdq_mboxes_sign(&mxm, 1); 6105 6106 mcx_cmdq_post(sc, cqe, 0); 6107 error = mcx_cmdq_poll(sc, cqe, 1000); 6108 if (error != 0) { 6109 printf("%s: query flow table timeout\n", DEVNAME(sc)); 6110 goto free; 6111 } 6112 error = mcx_cmdq_verify(cqe); 6113 if (error != 0) { 6114 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 6115 goto free; 6116 } 6117 6118 out = mcx_cmdq_out(cqe); 6119 switch (out->cmd_status) { 6120 case MCX_CQ_STATUS_OK: 6121 break; 6122 default: 6123 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 6124 out->cmd_status, betoh32(out->cmd_syndrome)); 6125 error = -1; 6126 goto free; 6127 } 6128 6129 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 6130 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6131 dump = (uint8_t *)mbout + 8; 6132 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 6133 printf("%.2x ", dump[i]); 6134 if (i % 16 == 15) 6135 printf("\n"); 6136 } 6137 free: 6138 mcx_cq_mboxes_free(sc, &mxm); 6139 return (error); 6140 } 6141 int 6142 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 6143 { 6144 struct mcx_dmamem mxm; 6145 struct mcx_cmdq_entry *cqe; 6146 struct mcx_cmd_query_flow_table_entry_in *in; 6147 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 6148 struct mcx_cmd_query_flow_table_entry_out *out; 6149 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 6150 uint8_t token = mcx_cmdq_token(sc); 6151 int error; 6152 int i; 6153 uint8_t *dump; 6154 6155 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6156 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6157 sizeof(*out) + sizeof(*mbout) + 16, token); 6158 6159 in = mcx_cmdq_in(cqe); 6160 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 6161 in->cmd_op_mod = htobe16(0); 6162 6163 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6164 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6165 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6166 &cqe->cq_output_ptr, token) != 0) { 6167 printf(", unable to allocate " 6168 "query flow table entry mailboxes\n"); 6169 return (-1); 6170 } 6171 cqe->cq_input_ptr = cqe->cq_output_ptr; 6172 6173 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6174 mbin->cmd_table_type = 0; 6175 mbin->cmd_table_id = htobe32(flow_table_id); 6176 mbin->cmd_flow_index = htobe32(index); 6177 6178 mcx_cmdq_mboxes_sign(&mxm, 1); 6179 6180 mcx_cmdq_post(sc, cqe, 0); 6181 error = mcx_cmdq_poll(sc, cqe, 1000); 6182 if (error != 0) { 6183 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 6184 goto free; 6185 } 6186 error = mcx_cmdq_verify(cqe); 6187 if (error != 0) { 6188 printf("%s: query flow table entry reply corrupt\n", 6189 DEVNAME(sc)); 6190 goto free; 6191 } 6192 6193 out = mcx_cmdq_out(cqe); 6194 switch (out->cmd_status) { 6195 case MCX_CQ_STATUS_OK: 6196 break; 6197 default: 6198 printf("%s: query flow table entry failed (%x/%x)\n", 6199 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6200 error = -1; 6201 goto free; 6202 } 6203 6204 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 6205 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6206 dump = (uint8_t *)mbout; 6207 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6208 printf("%.2x ", dump[i]); 6209 if (i % 16 == 15) 6210 printf("\n"); 6211 } 6212 6213 free: 6214 mcx_cq_mboxes_free(sc, &mxm); 6215 return (error); 6216 } 6217 6218 int 6219 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 6220 { 6221 struct mcx_dmamem mxm; 6222 struct mcx_cmdq_entry *cqe; 6223 struct mcx_cmd_query_flow_group_in *in; 6224 struct mcx_cmd_query_flow_group_mb_in *mbin; 6225 struct mcx_cmd_query_flow_group_out *out; 6226 struct mcx_cmd_query_flow_group_mb_out *mbout; 6227 uint8_t token = mcx_cmdq_token(sc); 6228 int error; 6229 int i; 6230 uint8_t *dump; 6231 6232 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6233 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6234 sizeof(*out) + sizeof(*mbout) + 16, token); 6235 6236 in = mcx_cmdq_in(cqe); 6237 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 6238 in->cmd_op_mod = htobe16(0); 6239 6240 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6241 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6242 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6243 &cqe->cq_output_ptr, token) != 0) { 6244 printf(", unable to allocate query flow group mailboxes\n"); 6245 return (-1); 6246 } 6247 cqe->cq_input_ptr = cqe->cq_output_ptr; 6248 6249 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6250 mbin->cmd_table_type = 0; 6251 mbin->cmd_table_id = htobe32(flow_table_id); 6252 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 6253 6254 mcx_cmdq_mboxes_sign(&mxm, 1); 6255 6256 mcx_cmdq_post(sc, cqe, 0); 6257 error = mcx_cmdq_poll(sc, cqe, 1000); 6258 if (error != 0) { 6259 printf("%s: query flow group timeout\n", DEVNAME(sc)); 6260 goto free; 6261 } 6262 error = mcx_cmdq_verify(cqe); 6263 if (error != 0) { 6264 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 6265 goto free; 6266 } 6267 6268 out = mcx_cmdq_out(cqe); 6269 switch (out->cmd_status) { 6270 case MCX_CQ_STATUS_OK: 6271 break; 6272 default: 6273 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 6274 out->cmd_status, betoh32(out->cmd_syndrome)); 6275 error = -1; 6276 goto free; 6277 } 6278 6279 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 6280 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6281 dump = (uint8_t *)mbout; 6282 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6283 printf("%.2x ", dump[i]); 6284 if (i % 16 == 15) 6285 printf("\n"); 6286 } 6287 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 6288 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6289 printf("%.2x ", dump[i]); 6290 if (i % 16 == 15) 6291 printf("\n"); 6292 } 6293 6294 free: 6295 mcx_cq_mboxes_free(sc, &mxm); 6296 return (error); 6297 } 6298 6299 static int 6300 mcx_dump_counters(struct mcx_softc *sc) 6301 { 6302 struct mcx_dmamem mxm; 6303 struct mcx_cmdq_entry *cqe; 6304 struct mcx_cmd_query_vport_counters_in *in; 6305 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6306 struct mcx_cmd_query_vport_counters_out *out; 6307 struct mcx_nic_vport_counters *counters; 6308 int error, token; 6309 6310 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6311 token = mcx_cmdq_token(sc); 6312 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6313 sizeof(*out) + sizeof(*counters), token); 6314 6315 in = mcx_cmdq_in(cqe); 6316 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6317 in->cmd_op_mod = htobe16(0); 6318 6319 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6320 &cqe->cq_output_ptr, token) != 0) { 6321 printf(", unable to allocate " 6322 "query nic vport counters mailboxen\n"); 6323 return (-1); 6324 } 6325 cqe->cq_input_ptr = cqe->cq_output_ptr; 6326 6327 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6328 mbin->cmd_clear = 0x80; 6329 6330 mcx_cmdq_mboxes_sign(&mxm, 1); 6331 mcx_cmdq_post(sc, cqe, 0); 6332 6333 error = mcx_cmdq_poll(sc, cqe, 1000); 6334 if (error != 0) { 6335 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6336 goto free; 6337 } 6338 if (mcx_cmdq_verify(cqe) != 0) { 6339 printf("%s: query nic vport counters command corrupt\n", 6340 DEVNAME(sc)); 6341 goto free; 6342 } 6343 6344 out = mcx_cmdq_out(cqe); 6345 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6346 printf("%s: query nic vport counters failed (%x, %x)\n", 6347 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6348 error = -1; 6349 goto free; 6350 } 6351 6352 counters = (struct mcx_nic_vport_counters *) 6353 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6354 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6355 counters->rx_ucast.packets + counters->tx_ucast.packets + 6356 counters->rx_err.packets + counters->tx_err.packets) 6357 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6358 DEVNAME(sc), 6359 betoh64(counters->tx_err.packets), 6360 betoh64(counters->rx_err.packets), 6361 betoh64(counters->tx_ucast.packets), 6362 betoh64(counters->rx_ucast.packets), 6363 betoh64(counters->tx_bcast.packets), 6364 betoh64(counters->rx_bcast.packets)); 6365 free: 6366 mcx_dmamem_free(sc, &mxm); 6367 6368 return (error); 6369 } 6370 6371 static int 6372 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6373 { 6374 struct mcx_dmamem mxm; 6375 struct mcx_cmdq_entry *cqe; 6376 struct mcx_cmd_query_flow_counter_in *in; 6377 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6378 struct mcx_cmd_query_flow_counter_out *out; 6379 struct mcx_counter *counters; 6380 int error, token; 6381 6382 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6383 token = mcx_cmdq_token(sc); 6384 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6385 sizeof(*counters), token); 6386 6387 in = mcx_cmdq_in(cqe); 6388 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6389 in->cmd_op_mod = htobe16(0); 6390 6391 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6392 &cqe->cq_output_ptr, token) != 0) { 6393 printf(", unable to allocate query flow counter mailboxen\n"); 6394 return (-1); 6395 } 6396 cqe->cq_input_ptr = cqe->cq_output_ptr; 6397 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6398 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6399 mbin->cmd_clear = 0x80; 6400 6401 mcx_cmdq_mboxes_sign(&mxm, 1); 6402 mcx_cmdq_post(sc, cqe, 0); 6403 6404 error = mcx_cmdq_poll(sc, cqe, 1000); 6405 if (error != 0) { 6406 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6407 goto free; 6408 } 6409 if (mcx_cmdq_verify(cqe) != 0) { 6410 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6411 goto free; 6412 } 6413 6414 out = mcx_cmdq_out(cqe); 6415 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6416 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6417 out->cmd_status, betoh32(out->cmd_syndrome)); 6418 error = -1; 6419 goto free; 6420 } 6421 6422 counters = (struct mcx_counter *) 6423 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6424 if (counters->packets) 6425 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6426 betoh64(counters->packets)); 6427 free: 6428 mcx_dmamem_free(sc, &mxm); 6429 6430 return (error); 6431 } 6432 6433 #endif 6434 6435 #if NKSTAT > 0 6436 6437 int 6438 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx) 6439 { 6440 struct mcx_dmamem mxm; 6441 struct mcx_cmdq_entry *cqe; 6442 struct mcx_cmd_query_rq_in *in; 6443 struct mcx_cmd_query_rq_out *out; 6444 struct mcx_cmd_query_rq_mb_out *mbout; 6445 uint8_t token = mcx_cmdq_token(sc); 6446 int error; 6447 6448 cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT); 6449 if (cqe == NULL) 6450 return (-1); 6451 6452 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6453 token); 6454 6455 in = mcx_cmdq_in(cqe); 6456 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 6457 in->cmd_op_mod = htobe16(0); 6458 in->cmd_rqn = htobe32(rx->rx_rqn); 6459 6460 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6461 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6462 &cqe->cq_output_ptr, token) != 0) { 6463 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc)); 6464 return (-1); 6465 } 6466 6467 mcx_cmdq_mboxes_sign(&mxm, 1); 6468 6469 error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000); 6470 if (error != 0) { 6471 printf("%s: query rq timeout\n", DEVNAME(sc)); 6472 goto free; 6473 } 6474 error = mcx_cmdq_verify(cqe); 6475 if (error != 0) { 6476 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 6477 goto free; 6478 } 6479 6480 out = mcx_cmdq_out(cqe); 6481 switch (out->cmd_status) { 6482 case MCX_CQ_STATUS_OK: 6483 break; 6484 default: 6485 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6486 out->cmd_status, betoh32(out->cmd_syndrome)); 6487 error = -1; 6488 goto free; 6489 } 6490 6491 mbout = (struct mcx_cmd_query_rq_mb_out *) 6492 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6493 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx)); 6494 6495 free: 6496 mcx_cq_mboxes_free(sc, &mxm); 6497 return (error); 6498 } 6499 6500 int 6501 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx) 6502 { 6503 struct mcx_dmamem mxm; 6504 struct mcx_cmdq_entry *cqe; 6505 struct mcx_cmd_query_sq_in *in; 6506 struct mcx_cmd_query_sq_out *out; 6507 struct mcx_cmd_query_sq_mb_out *mbout; 6508 uint8_t token = mcx_cmdq_token(sc); 6509 int error; 6510 6511 cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT); 6512 if (cqe == NULL) 6513 return (-1); 6514 6515 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6516 token); 6517 6518 in = mcx_cmdq_in(cqe); 6519 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6520 in->cmd_op_mod = htobe16(0); 6521 in->cmd_sqn = htobe32(tx->tx_sqn); 6522 6523 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6524 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6525 &cqe->cq_output_ptr, token) != 0) { 6526 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc)); 6527 return (-1); 6528 } 6529 6530 mcx_cmdq_mboxes_sign(&mxm, 1); 6531 6532 error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000); 6533 if (error != 0) { 6534 printf("%s: query sq timeout\n", DEVNAME(sc)); 6535 goto free; 6536 } 6537 error = mcx_cmdq_verify(cqe); 6538 if (error != 0) { 6539 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6540 goto free; 6541 } 6542 6543 out = mcx_cmdq_out(cqe); 6544 switch (out->cmd_status) { 6545 case MCX_CQ_STATUS_OK: 6546 break; 6547 default: 6548 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6549 out->cmd_status, betoh32(out->cmd_syndrome)); 6550 error = -1; 6551 goto free; 6552 } 6553 6554 mbout = (struct mcx_cmd_query_sq_mb_out *) 6555 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6556 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx)); 6557 6558 free: 6559 mcx_cq_mboxes_free(sc, &mxm); 6560 return (error); 6561 } 6562 6563 int 6564 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx) 6565 { 6566 struct mcx_dmamem mxm; 6567 struct mcx_cmdq_entry *cqe; 6568 struct mcx_cmd_query_cq_in *in; 6569 struct mcx_cmd_query_cq_out *out; 6570 struct mcx_cq_ctx *ctx; 6571 uint8_t token = mcx_cmdq_token(sc); 6572 int error; 6573 6574 cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT); 6575 if (cqe == NULL) 6576 return (-1); 6577 6578 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6579 token); 6580 6581 in = mcx_cmdq_in(cqe); 6582 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ); 6583 in->cmd_op_mod = htobe16(0); 6584 in->cmd_cqn = htobe32(cq->cq_n); 6585 6586 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6587 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6588 &cqe->cq_output_ptr, token) != 0) { 6589 printf("%s: unable to allocate query cq mailboxes\n", DEVNAME(sc)); 6590 return (-1); 6591 } 6592 6593 mcx_cmdq_mboxes_sign(&mxm, 1); 6594 6595 error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000); 6596 if (error != 0) { 6597 printf("%s: query cq timeout\n", DEVNAME(sc)); 6598 goto free; 6599 } 6600 error = mcx_cmdq_verify(cqe); 6601 if (error != 0) { 6602 printf("%s: query cq reply corrupt\n", DEVNAME(sc)); 6603 goto free; 6604 } 6605 6606 out = mcx_cmdq_out(cqe); 6607 switch (out->cmd_status) { 6608 case MCX_CQ_STATUS_OK: 6609 break; 6610 default: 6611 printf("%s: query cq failed (%x/%x)\n", DEVNAME(sc), 6612 out->cmd_status, betoh32(out->cmd_syndrome)); 6613 error = -1; 6614 goto free; 6615 } 6616 6617 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6618 memcpy(cq_ctx, ctx, sizeof(*cq_ctx)); 6619 free: 6620 mcx_cq_mboxes_free(sc, &mxm); 6621 return (error); 6622 } 6623 6624 int 6625 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx) 6626 { 6627 struct mcx_dmamem mxm; 6628 struct mcx_cmdq_entry *cqe; 6629 struct mcx_cmd_query_eq_in *in; 6630 struct mcx_cmd_query_eq_out *out; 6631 struct mcx_eq_ctx *ctx; 6632 uint8_t token = mcx_cmdq_token(sc); 6633 int error; 6634 6635 cqe = mcx_get_cmdq_entry(sc, MCX_CMDQ_SLOT_KSTAT); 6636 if (cqe == NULL) 6637 return (-1); 6638 6639 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6640 token); 6641 6642 in = mcx_cmdq_in(cqe); 6643 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ); 6644 in->cmd_op_mod = htobe16(0); 6645 in->cmd_eqn = htobe32(eq->eq_n); 6646 6647 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6648 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6649 &cqe->cq_output_ptr, token) != 0) { 6650 printf("%s: unable to allocate query eq mailboxes\n", DEVNAME(sc)); 6651 return (-1); 6652 } 6653 6654 mcx_cmdq_mboxes_sign(&mxm, 1); 6655 6656 error = mcx_cmdq_exec(sc, cqe, MCX_CMDQ_SLOT_KSTAT, 1000); 6657 if (error != 0) { 6658 printf("%s: query eq timeout\n", DEVNAME(sc)); 6659 goto free; 6660 } 6661 error = mcx_cmdq_verify(cqe); 6662 if (error != 0) { 6663 printf("%s: query eq reply corrupt\n", DEVNAME(sc)); 6664 goto free; 6665 } 6666 6667 out = mcx_cmdq_out(cqe); 6668 switch (out->cmd_status) { 6669 case MCX_CQ_STATUS_OK: 6670 break; 6671 default: 6672 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc), 6673 out->cmd_status, betoh32(out->cmd_syndrome)); 6674 error = -1; 6675 goto free; 6676 } 6677 6678 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6679 memcpy(eq_ctx, ctx, sizeof(*eq_ctx)); 6680 free: 6681 mcx_cq_mboxes_free(sc, &mxm); 6682 return (error); 6683 } 6684 6685 #endif /* NKSTAT > 0 */ 6686 6687 static inline unsigned int 6688 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots) 6689 { 6690 struct mcx_rq_entry *ring, *rqe; 6691 struct mcx_slot *ms; 6692 struct mbuf *m; 6693 uint slot, p, fills; 6694 6695 ring = MCX_DMA_KVA(&rx->rx_rq_mem); 6696 p = rx->rx_prod; 6697 6698 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6699 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 6700 6701 for (fills = 0; fills < nslots; fills++) { 6702 slot = p % (1 << MCX_LOG_RQ_SIZE); 6703 6704 ms = &rx->rx_slots[slot]; 6705 rqe = &ring[slot]; 6706 6707 m = MCLGETL(NULL, M_DONTWAIT, sc->sc_rxbufsz); 6708 if (m == NULL) 6709 break; 6710 6711 m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz); 6712 m->m_data += ETHER_ALIGN; 6713 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6714 6715 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6716 BUS_DMA_NOWAIT) != 0) { 6717 m_freem(m); 6718 break; 6719 } 6720 ms->ms_m = m; 6721 6722 htobem32(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len); 6723 htobem64(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr); 6724 htobem32(&rqe->rqe_lkey, sc->sc_lkey); 6725 6726 p++; 6727 } 6728 6729 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6730 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 6731 6732 rx->rx_prod = p; 6733 6734 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6735 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 6736 htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell), 6737 p & MCX_WQ_DOORBELL_MASK); 6738 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6739 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 6740 6741 return (nslots - fills); 6742 } 6743 6744 int 6745 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6746 { 6747 u_int slots; 6748 6749 slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6750 if (slots == 0) 6751 return (1); 6752 6753 slots = mcx_rx_fill_slots(sc, rx, slots); 6754 if_rxr_put(&rx->rx_rxr, slots); 6755 return (0); 6756 } 6757 6758 void 6759 mcx_refill(void *xrx) 6760 { 6761 struct mcx_rx *rx = xrx; 6762 struct mcx_softc *sc = rx->rx_softc; 6763 6764 mcx_rx_fill(sc, rx); 6765 6766 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6767 timeout_add(&rx->rx_refill, 1); 6768 } 6769 6770 static int 6771 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6772 struct mcx_cq_entry *cqe) 6773 { 6774 struct mcx_slot *ms; 6775 bus_dmamap_t map; 6776 int slot, slots; 6777 6778 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6779 6780 ms = &tx->tx_slots[slot]; 6781 map = ms->ms_map; 6782 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6783 BUS_DMASYNC_POSTWRITE); 6784 6785 slots = 1; 6786 if (map->dm_nsegs > 1) 6787 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6788 6789 bus_dmamap_unload(sc->sc_dmat, map); 6790 m_freem(ms->ms_m); 6791 ms->ms_m = NULL; 6792 6793 return (slots); 6794 } 6795 6796 static uint64_t 6797 mcx_uptime(void) 6798 { 6799 struct timespec ts; 6800 6801 nanouptime(&ts); 6802 6803 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6804 } 6805 6806 static void 6807 mcx_calibrate_first(struct mcx_softc *sc) 6808 { 6809 struct mcx_calibration *c = &sc->sc_calibration[0]; 6810 int s; 6811 6812 sc->sc_calibration_gen = 0; 6813 6814 s = splhigh(); /* crit_enter? */ 6815 c->c_ubase = mcx_uptime(); 6816 c->c_tbase = mcx_timer(sc); 6817 splx(s); 6818 c->c_ratio = 0; 6819 6820 #ifdef notyet 6821 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 6822 #endif 6823 } 6824 6825 #define MCX_TIMESTAMP_SHIFT 24 6826 6827 static void 6828 mcx_calibrate(void *arg) 6829 { 6830 struct mcx_softc *sc = arg; 6831 struct mcx_calibration *nc, *pc; 6832 uint64_t udiff, tdiff; 6833 unsigned int gen; 6834 int s; 6835 6836 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 6837 return; 6838 6839 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 6840 6841 gen = sc->sc_calibration_gen; 6842 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6843 gen++; 6844 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6845 6846 nc->c_uptime = pc->c_ubase; 6847 nc->c_timestamp = pc->c_tbase; 6848 6849 s = splhigh(); /* crit_enter? */ 6850 nc->c_ubase = mcx_uptime(); 6851 nc->c_tbase = mcx_timer(sc); 6852 splx(s); 6853 6854 udiff = nc->c_ubase - nc->c_uptime; 6855 tdiff = nc->c_tbase - nc->c_timestamp; 6856 6857 /* 6858 * udiff is the wall clock time between calibration ticks, 6859 * which should be 32 seconds or 32 billion nanoseconds. if 6860 * we squint, 1 billion nanoseconds is kind of like a 32 bit 6861 * number, so 32 billion should still have a lot of high bits 6862 * spare. we use this space by shifting the nanoseconds up 6863 * 24 bits so we have a nice big number to divide by the 6864 * number of mcx timer ticks. 6865 */ 6866 nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff; 6867 6868 membar_producer(); 6869 sc->sc_calibration_gen = gen; 6870 } 6871 6872 static int 6873 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6874 struct mcx_cq_entry *cqe, struct mbuf_list *ml, 6875 const struct mcx_calibration *c) 6876 { 6877 struct mcx_slot *ms; 6878 struct mbuf *m; 6879 uint32_t flags, len; 6880 int slot; 6881 6882 len = bemtoh32(&cqe->cq_byte_cnt); 6883 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6884 6885 ms = &rx->rx_slots[slot]; 6886 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD); 6887 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6888 6889 m = ms->ms_m; 6890 ms->ms_m = NULL; 6891 6892 m->m_pkthdr.len = m->m_len = len; 6893 6894 if (cqe->cq_rx_hash_type) { 6895 m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); 6896 m->m_pkthdr.csum_flags |= M_FLOWID; 6897 } 6898 6899 flags = bemtoh32(&cqe->cq_flags); 6900 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) 6901 m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK; 6902 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) 6903 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK | 6904 M_UDP_CSUM_IN_OK; 6905 #if NVLAN > 0 6906 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6907 m->m_pkthdr.ether_vtag = (flags & 6908 MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6909 m->m_flags |= M_VLANTAG; 6910 } 6911 #endif 6912 6913 #ifdef notyet 6914 if (ISSET(sc->sc_ac.ac_if.if_flags, IFF_LINK0) && c->c_ratio) { 6915 uint64_t t = bemtoh64(&cqe->cq_timestamp); 6916 t -= c->c_timestamp; 6917 t *= c->c_ratio; 6918 t >>= MCX_TIMESTAMP_SHIFT; 6919 t += c->c_uptime; 6920 6921 m->m_pkthdr.ph_timestamp = t; 6922 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6923 } 6924 #endif 6925 6926 ml_enqueue(ml, m); 6927 6928 return (1); 6929 } 6930 6931 static struct mcx_cq_entry * 6932 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6933 { 6934 struct mcx_cq_entry *cqe; 6935 int next; 6936 6937 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 6938 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 6939 6940 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 6941 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 6942 return (&cqe[next]); 6943 } 6944 6945 return (NULL); 6946 } 6947 6948 static void 6949 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 6950 { 6951 struct mcx_cq_doorbell *db; 6952 bus_size_t offset; 6953 uint32_t val; 6954 uint64_t uval; 6955 6956 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 6957 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6958 6959 db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell); 6960 6961 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6962 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE); 6963 6964 htobem32(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6965 htobem32(&db->db_arm_ci, val); 6966 6967 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6968 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE); 6969 6970 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL; 6971 6972 uval = (uint64_t)val << 32; 6973 uval |= cq->cq_n; 6974 6975 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval)); 6976 mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE); 6977 } 6978 6979 void 6980 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 6981 { 6982 struct mcx_rx *rx = &q->q_rx; 6983 struct mcx_tx *tx = &q->q_tx; 6984 const struct mcx_calibration *c; 6985 unsigned int gen; 6986 struct mcx_cq_entry *cqe; 6987 uint8_t *cqp; 6988 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 6989 int rxfree, txfree; 6990 6991 gen = sc->sc_calibration_gen; 6992 membar_consumer(); 6993 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6994 6995 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 6996 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 6997 6998 rxfree = 0; 6999 txfree = 0; 7000 while ((cqe = mcx_next_cq_entry(sc, cq))) { 7001 uint8_t opcode; 7002 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 7003 switch (opcode) { 7004 case MCX_CQ_ENTRY_OPCODE_REQ: 7005 txfree += mcx_process_txeof(sc, tx, cqe); 7006 break; 7007 case MCX_CQ_ENTRY_OPCODE_SEND: 7008 rxfree += mcx_process_rx(sc, rx, cqe, &ml, c); 7009 break; 7010 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 7011 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 7012 cqp = (uint8_t *)cqe; 7013 /* printf("%s: cq completion error: %x\n", 7014 DEVNAME(sc), cqp[0x37]); */ 7015 break; 7016 7017 default: 7018 /* printf("%s: cq completion opcode %x??\n", 7019 DEVNAME(sc), opcode); */ 7020 break; 7021 } 7022 7023 cq->cq_cons++; 7024 } 7025 7026 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 7027 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 7028 7029 if (rxfree > 0) { 7030 if_rxr_put(&rx->rx_rxr, rxfree); 7031 if (ifiq_input(rx->rx_ifiq, &ml)) 7032 if_rxr_livelocked(&rx->rx_rxr); 7033 7034 mcx_rx_fill(sc, rx); 7035 if (if_rxr_inuse(&rx->rx_rxr) == 0) 7036 timeout_add(&rx->rx_refill, 1); 7037 } 7038 7039 cq->cq_count++; 7040 mcx_arm_cq(sc, cq, q->q_uar); 7041 7042 if (txfree > 0) { 7043 tx->tx_cons += txfree; 7044 if (ifq_is_oactive(tx->tx_ifq)) 7045 ifq_restart(tx->tx_ifq); 7046 } 7047 } 7048 7049 7050 static void 7051 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 7052 { 7053 bus_size_t offset; 7054 uint32_t val; 7055 7056 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 7057 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 7058 7059 mcx_wr(sc, offset, val); 7060 mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE); 7061 } 7062 7063 static struct mcx_eq_entry * 7064 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 7065 { 7066 struct mcx_eq_entry *eqe; 7067 int next; 7068 7069 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 7070 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 7071 if ((eqe[next].eq_owner & 1) == 7072 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 7073 eq->eq_cons++; 7074 return (&eqe[next]); 7075 } 7076 return (NULL); 7077 } 7078 7079 int 7080 mcx_admin_intr(void *xsc) 7081 { 7082 struct mcx_softc *sc = (struct mcx_softc *)xsc; 7083 struct mcx_eq *eq = &sc->sc_admin_eq; 7084 struct mcx_eq_entry *eqe; 7085 7086 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7087 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 7088 7089 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 7090 switch (eqe->eq_event_type) { 7091 case MCX_EVENT_TYPE_LAST_WQE: 7092 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 7093 break; 7094 7095 case MCX_EVENT_TYPE_CQ_ERROR: 7096 /* printf("%s: cq error\n", DEVNAME(sc)); */ 7097 break; 7098 7099 case MCX_EVENT_TYPE_CMD_COMPLETION: 7100 mtx_enter(&sc->sc_cmdq_mtx); 7101 wakeup(&sc->sc_cmdq_token); 7102 mtx_leave(&sc->sc_cmdq_mtx); 7103 break; 7104 7105 case MCX_EVENT_TYPE_PORT_CHANGE: 7106 task_add(systq, &sc->sc_port_change); 7107 break; 7108 7109 default: 7110 /* printf("%s: something happened\n", DEVNAME(sc)); */ 7111 break; 7112 } 7113 } 7114 7115 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7116 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 7117 7118 mcx_arm_eq(sc, eq, sc->sc_uar); 7119 7120 return (1); 7121 } 7122 7123 int 7124 mcx_cq_intr(void *xq) 7125 { 7126 struct mcx_queues *q = (struct mcx_queues *)xq; 7127 struct mcx_softc *sc = q->q_sc; 7128 struct mcx_eq *eq = &q->q_eq; 7129 struct mcx_eq_entry *eqe; 7130 int cqn; 7131 7132 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7133 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 7134 7135 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 7136 switch (eqe->eq_event_type) { 7137 case MCX_EVENT_TYPE_COMPLETION: 7138 cqn = betoh32(eqe->eq_event_data[6]); 7139 if (cqn == q->q_cq.cq_n) 7140 mcx_process_cq(sc, q, &q->q_cq); 7141 break; 7142 } 7143 } 7144 7145 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7146 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 7147 7148 mcx_arm_eq(sc, eq, q->q_uar); 7149 7150 return (1); 7151 } 7152 7153 static void 7154 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 7155 int total) 7156 { 7157 struct mcx_slot *ms; 7158 7159 int i = allocated; 7160 while (i-- > 0) { 7161 ms = &slots[i]; 7162 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 7163 if (ms->ms_m != NULL) 7164 m_freem(ms->ms_m); 7165 } 7166 free(slots, M_DEVBUF, total * sizeof(*ms)); 7167 } 7168 7169 static int 7170 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 7171 { 7172 struct mcx_rx *rx; 7173 struct mcx_tx *tx; 7174 struct mcx_slot *ms; 7175 int i; 7176 7177 rx = &q->q_rx; 7178 rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 7179 M_DEVBUF, M_WAITOK | M_ZERO); 7180 if (rx->rx_slots == NULL) { 7181 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 7182 return ENOMEM; 7183 } 7184 7185 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 7186 ms = &rx->rx_slots[i]; 7187 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 7188 sc->sc_hardmtu, 0, 7189 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7190 &ms->ms_map) != 0) { 7191 printf("%s: failed to allocate rx dma maps\n", 7192 DEVNAME(sc)); 7193 goto destroy_rx_slots; 7194 } 7195 } 7196 7197 tx = &q->q_tx; 7198 tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 7199 M_DEVBUF, M_WAITOK | M_ZERO); 7200 if (tx->tx_slots == NULL) { 7201 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 7202 goto destroy_rx_slots; 7203 } 7204 7205 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 7206 ms = &tx->tx_slots[i]; 7207 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 7208 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 7209 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7210 &ms->ms_map) != 0) { 7211 printf("%s: failed to allocate tx dma maps\n", 7212 DEVNAME(sc)); 7213 goto destroy_tx_slots; 7214 } 7215 } 7216 7217 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 7218 q->q_eq.eq_n) != 0) 7219 goto destroy_tx_slots; 7220 7221 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 7222 != 0) 7223 goto destroy_cq; 7224 7225 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 7226 goto destroy_sq; 7227 7228 return 0; 7229 7230 destroy_sq: 7231 mcx_destroy_sq(sc, tx); 7232 destroy_cq: 7233 mcx_destroy_cq(sc, &q->q_cq); 7234 destroy_tx_slots: 7235 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 7236 tx->tx_slots = NULL; 7237 7238 i = (1 << MCX_LOG_RQ_SIZE); 7239 destroy_rx_slots: 7240 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 7241 rx->rx_slots = NULL; 7242 return ENOMEM; 7243 } 7244 7245 static int 7246 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 7247 { 7248 int i; 7249 int count; 7250 7251 count = 0; 7252 for (i = 0; i < nitems(mcx_rss_config); i++) { 7253 if (mcx_rss_config[i].flow_group == group) 7254 count++; 7255 } 7256 7257 return count; 7258 } 7259 7260 static int 7261 mcx_up(struct mcx_softc *sc) 7262 { 7263 struct ifnet *ifp = &sc->sc_ac.ac_if; 7264 struct mcx_rx *rx; 7265 struct mcx_tx *tx; 7266 int i, start, count, flow_group, flow_index; 7267 struct mcx_flow_match match_crit; 7268 struct mcx_rss_rule *rss; 7269 uint32_t dest; 7270 int rqns[MCX_MAX_QUEUES]; 7271 7272 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 7273 goto down; 7274 7275 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 7276 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 7277 goto down; 7278 } 7279 } 7280 7281 /* RSS flow table and flow groups */ 7282 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 7283 &sc->sc_rss_flow_table_id) != 0) 7284 goto down; 7285 7286 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7287 sc->sc_rss_flow_table_id; 7288 7289 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 7290 memset(&match_crit, 0, sizeof(match_crit)); 7291 match_crit.mc_ethertype = 0xffff; 7292 match_crit.mc_ip_proto = 0xff; 7293 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 7294 start = 0; 7295 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 7296 if (count != 0) { 7297 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7298 MCX_FLOW_GROUP_RSS_L4, start, count, 7299 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7300 goto down; 7301 start += count; 7302 } 7303 7304 /* L3 RSS flow group (v4/v6, including fragments) */ 7305 memset(&match_crit, 0, sizeof(match_crit)); 7306 match_crit.mc_ethertype = 0xffff; 7307 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 7308 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7309 MCX_FLOW_GROUP_RSS_L3, start, count, 7310 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7311 goto down; 7312 start += count; 7313 7314 /* non-RSS flow group */ 7315 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 7316 memset(&match_crit, 0, sizeof(match_crit)); 7317 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7318 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 7319 goto down; 7320 7321 /* Root flow table, matching packets based on mac address */ 7322 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 7323 &sc->sc_mac_flow_table_id) != 0) 7324 goto down; 7325 7326 /* promisc flow group */ 7327 start = 0; 7328 memset(&match_crit, 0, sizeof(match_crit)); 7329 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7330 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 7331 goto down; 7332 sc->sc_promisc_flow_enabled = 0; 7333 start++; 7334 7335 /* all multicast flow group */ 7336 match_crit.mc_dest_mac[0] = 0x01; 7337 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7338 MCX_FLOW_GROUP_ALLMULTI, start, 1, 7339 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7340 goto down; 7341 sc->sc_allmulti_flow_enabled = 0; 7342 start++; 7343 7344 /* mac address matching flow group */ 7345 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 7346 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7347 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 7348 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7349 goto down; 7350 7351 /* flow table entries for unicast and broadcast */ 7352 start = 0; 7353 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7354 sc->sc_ac.ac_enaddr, dest) != 0) 7355 goto down; 7356 start++; 7357 7358 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7359 etherbroadcastaddr, dest) != 0) 7360 goto down; 7361 start++; 7362 7363 /* multicast entries go after that */ 7364 sc->sc_mcast_flow_base = start; 7365 7366 /* re-add any existing multicast flows */ 7367 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7368 if (sc->sc_mcast_flows[i][0] != 0) { 7369 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 7370 sc->sc_mcast_flow_base + i, 7371 sc->sc_mcast_flows[i], dest); 7372 } 7373 } 7374 7375 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 7376 goto down; 7377 7378 /* 7379 * the RQT can be any size as long as it's a power of two. 7380 * since we also restrict the number of queues to a power of two, 7381 * we can just put each rx queue in once. 7382 */ 7383 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) 7384 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 7385 7386 if (mcx_create_rqt(sc, intrmap_count(sc->sc_intrmap), rqns, 7387 &sc->sc_rqt) != 0) 7388 goto down; 7389 7390 start = 0; 7391 flow_index = 0; 7392 flow_group = -1; 7393 for (i = 0; i < nitems(mcx_rss_config); i++) { 7394 rss = &mcx_rss_config[i]; 7395 if (rss->flow_group != flow_group) { 7396 flow_group = rss->flow_group; 7397 flow_index = 0; 7398 } 7399 7400 if (rss->hash_sel == 0) { 7401 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 7402 &sc->sc_tir[i]) != 0) 7403 goto down; 7404 } else { 7405 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 7406 rss->hash_sel, &sc->sc_tir[i]) != 0) 7407 goto down; 7408 } 7409 7410 if (mcx_set_flow_table_entry_proto(sc, flow_group, 7411 flow_index, rss->ethertype, rss->ip_proto, 7412 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 7413 goto down; 7414 flow_index++; 7415 } 7416 7417 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 7418 struct mcx_queues *q = &sc->sc_queues[i]; 7419 rx = &q->q_rx; 7420 tx = &q->q_tx; 7421 7422 /* start the queues */ 7423 if (mcx_ready_sq(sc, tx) != 0) 7424 goto down; 7425 7426 if (mcx_ready_rq(sc, rx) != 0) 7427 goto down; 7428 7429 if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 7430 rx->rx_prod = 0; 7431 mcx_rx_fill(sc, rx); 7432 7433 tx->tx_cons = 0; 7434 tx->tx_prod = 0; 7435 ifq_clr_oactive(tx->tx_ifq); 7436 } 7437 7438 mcx_calibrate_first(sc); 7439 7440 SET(ifp->if_flags, IFF_RUNNING); 7441 7442 return ENETRESET; 7443 down: 7444 mcx_down(sc); 7445 return ENOMEM; 7446 } 7447 7448 static void 7449 mcx_down(struct mcx_softc *sc) 7450 { 7451 struct ifnet *ifp = &sc->sc_ac.ac_if; 7452 struct mcx_rss_rule *rss; 7453 int group, i, flow_group, flow_index; 7454 7455 CLR(ifp->if_flags, IFF_RUNNING); 7456 7457 /* 7458 * delete flow table entries first, so no packets can arrive 7459 * after the barriers 7460 */ 7461 if (sc->sc_promisc_flow_enabled) 7462 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 7463 if (sc->sc_allmulti_flow_enabled) 7464 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 7465 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 7466 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 7467 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7468 if (sc->sc_mcast_flows[i][0] != 0) { 7469 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 7470 sc->sc_mcast_flow_base + i); 7471 } 7472 } 7473 7474 flow_group = -1; 7475 flow_index = 0; 7476 for (i = 0; i < nitems(mcx_rss_config); i++) { 7477 rss = &mcx_rss_config[i]; 7478 if (rss->flow_group != flow_group) { 7479 flow_group = rss->flow_group; 7480 flow_index = 0; 7481 } 7482 7483 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 7484 7485 mcx_destroy_tir(sc, sc->sc_tir[i]); 7486 sc->sc_tir[i] = 0; 7487 7488 flow_index++; 7489 } 7490 intr_barrier(sc->sc_ihc); 7491 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 7492 struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; 7493 ifq_barrier(ifq); 7494 7495 timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill); 7496 7497 intr_barrier(sc->sc_queues[i].q_ihc); 7498 } 7499 7500 timeout_del_barrier(&sc->sc_calibrate); 7501 7502 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 7503 if (sc->sc_flow_group[group].g_id != -1) 7504 mcx_destroy_flow_group(sc, group); 7505 } 7506 7507 if (sc->sc_mac_flow_table_id != -1) { 7508 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 7509 sc->sc_mac_flow_table_id = -1; 7510 } 7511 if (sc->sc_rss_flow_table_id != -1) { 7512 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 7513 sc->sc_rss_flow_table_id = -1; 7514 } 7515 if (sc->sc_rqt != -1) { 7516 mcx_destroy_rqt(sc, sc->sc_rqt); 7517 sc->sc_rqt = -1; 7518 } 7519 7520 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 7521 struct mcx_queues *q = &sc->sc_queues[i]; 7522 struct mcx_rx *rx = &q->q_rx; 7523 struct mcx_tx *tx = &q->q_tx; 7524 struct mcx_cq *cq = &q->q_cq; 7525 7526 if (rx->rx_rqn != 0) 7527 mcx_destroy_rq(sc, rx); 7528 7529 if (tx->tx_sqn != 0) 7530 mcx_destroy_sq(sc, tx); 7531 7532 if (tx->tx_slots != NULL) { 7533 mcx_free_slots(sc, tx->tx_slots, 7534 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7535 tx->tx_slots = NULL; 7536 } 7537 if (rx->rx_slots != NULL) { 7538 mcx_free_slots(sc, rx->rx_slots, 7539 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7540 rx->rx_slots = NULL; 7541 } 7542 7543 if (cq->cq_n != 0) 7544 mcx_destroy_cq(sc, cq); 7545 } 7546 if (sc->sc_tis != 0) { 7547 mcx_destroy_tis(sc, sc->sc_tis); 7548 sc->sc_tis = 0; 7549 } 7550 } 7551 7552 static int 7553 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 7554 { 7555 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7556 struct ifreq *ifr = (struct ifreq *)data; 7557 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7558 int s, i, error = 0; 7559 uint32_t dest; 7560 7561 s = splnet(); 7562 switch (cmd) { 7563 case SIOCSIFADDR: 7564 ifp->if_flags |= IFF_UP; 7565 /* FALLTHROUGH */ 7566 7567 case SIOCSIFFLAGS: 7568 if (ISSET(ifp->if_flags, IFF_UP)) { 7569 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7570 error = ENETRESET; 7571 else 7572 error = mcx_up(sc); 7573 } else { 7574 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7575 mcx_down(sc); 7576 } 7577 break; 7578 7579 case SIOCGIFMEDIA: 7580 case SIOCSIFMEDIA: 7581 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 7582 break; 7583 7584 case SIOCGIFSFFPAGE: 7585 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 7586 break; 7587 7588 case SIOCGIFRXR: 7589 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 7590 break; 7591 7592 case SIOCADDMULTI: 7593 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 7594 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7595 if (error != 0) 7596 break; 7597 7598 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7599 sc->sc_rss_flow_table_id; 7600 7601 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7602 if (sc->sc_mcast_flows[i][0] == 0) { 7603 memcpy(sc->sc_mcast_flows[i], addrlo, 7604 ETHER_ADDR_LEN); 7605 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7606 mcx_set_flow_table_entry_mac(sc, 7607 MCX_FLOW_GROUP_MAC, 7608 sc->sc_mcast_flow_base + i, 7609 sc->sc_mcast_flows[i], dest); 7610 } 7611 break; 7612 } 7613 } 7614 7615 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7616 if (i == MCX_NUM_MCAST_FLOWS) { 7617 SET(ifp->if_flags, IFF_ALLMULTI); 7618 sc->sc_extra_mcast++; 7619 error = ENETRESET; 7620 } 7621 7622 if (sc->sc_ac.ac_multirangecnt > 0) { 7623 SET(ifp->if_flags, IFF_ALLMULTI); 7624 error = ENETRESET; 7625 } 7626 } 7627 } 7628 break; 7629 7630 case SIOCDELMULTI: 7631 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 7632 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7633 if (error != 0) 7634 break; 7635 7636 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7637 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7638 ETHER_ADDR_LEN) == 0) { 7639 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7640 mcx_delete_flow_table_entry(sc, 7641 MCX_FLOW_GROUP_MAC, 7642 sc->sc_mcast_flow_base + i); 7643 } 7644 sc->sc_mcast_flows[i][0] = 0; 7645 break; 7646 } 7647 } 7648 7649 if (i == MCX_NUM_MCAST_FLOWS) 7650 sc->sc_extra_mcast--; 7651 7652 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7653 (sc->sc_extra_mcast == 0) && 7654 (sc->sc_ac.ac_multirangecnt == 0)) { 7655 CLR(ifp->if_flags, IFF_ALLMULTI); 7656 error = ENETRESET; 7657 } 7658 } 7659 break; 7660 7661 default: 7662 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 7663 } 7664 7665 if (error == ENETRESET) { 7666 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7667 (IFF_UP | IFF_RUNNING)) 7668 mcx_iff(sc); 7669 error = 0; 7670 } 7671 splx(s); 7672 7673 return (error); 7674 } 7675 7676 static int 7677 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7678 { 7679 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7680 struct mcx_reg_mcia mcia; 7681 struct mcx_reg_pmlp pmlp; 7682 int offset, error; 7683 7684 rw_enter_write(&sc->sc_cmdq_ioctl_lk); 7685 7686 /* get module number */ 7687 memset(&pmlp, 0, sizeof(pmlp)); 7688 pmlp.rp_local_port = 1; 7689 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7690 sizeof(pmlp), MCX_CMDQ_SLOT_IOCTL); 7691 if (error != 0) { 7692 printf("%s: unable to get eeprom module number\n", 7693 DEVNAME(sc)); 7694 goto out; 7695 } 7696 7697 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7698 memset(&mcia, 0, sizeof(mcia)); 7699 mcia.rm_l = 0; 7700 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 7701 MCX_PMLP_MODULE_NUM_MASK; 7702 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7703 mcia.rm_page_num = sff->sff_page; 7704 mcia.rm_dev_addr = htobe16(offset); 7705 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7706 7707 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7708 &mcia, sizeof(mcia), MCX_CMDQ_SLOT_IOCTL); 7709 if (error != 0) { 7710 printf("%s: unable to read eeprom at %x\n", 7711 DEVNAME(sc), offset); 7712 goto out; 7713 } 7714 7715 memcpy(sff->sff_data + offset, mcia.rm_data, 7716 MCX_MCIA_EEPROM_BYTES); 7717 } 7718 7719 out: 7720 rw_exit_write(&sc->sc_cmdq_ioctl_lk); 7721 return (error); 7722 } 7723 7724 static int 7725 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 7726 { 7727 struct if_rxring_info *ifrs; 7728 unsigned int i; 7729 int error; 7730 7731 ifrs = mallocarray(intrmap_count(sc->sc_intrmap), sizeof(*ifrs), 7732 M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL); 7733 if (ifrs == NULL) 7734 return (ENOMEM); 7735 7736 for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { 7737 struct mcx_rx *rx = &sc->sc_queues[i].q_rx; 7738 struct if_rxring_info *ifr = &ifrs[i]; 7739 7740 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 7741 ifr->ifr_size = sc->sc_hardmtu; 7742 ifr->ifr_info = rx->rx_rxr; 7743 } 7744 7745 error = if_rxr_info_ioctl(ifri, i, ifrs); 7746 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 7747 7748 return (error); 7749 } 7750 7751 int 7752 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7753 { 7754 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7755 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7756 case 0: 7757 break; 7758 7759 case EFBIG: 7760 if (m_defrag(m, M_DONTWAIT) == 0 && 7761 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7762 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7763 break; 7764 7765 default: 7766 return (1); 7767 } 7768 7769 ms->ms_m = m; 7770 return (0); 7771 } 7772 7773 static void 7774 mcx_start(struct ifqueue *ifq) 7775 { 7776 struct mcx_tx *tx = ifq->ifq_softc; 7777 struct ifnet *ifp = ifq->ifq_if; 7778 struct mcx_softc *sc = ifp->if_softc; 7779 struct mcx_sq_entry *sq, *sqe; 7780 struct mcx_sq_entry_seg *sqs; 7781 struct mcx_slot *ms; 7782 bus_dmamap_t map; 7783 struct mbuf *m; 7784 u_int idx, free, used; 7785 uint64_t *bf; 7786 uint32_t csum; 7787 size_t bf_base; 7788 int i, seg, nseg; 7789 7790 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7791 7792 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7793 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7794 7795 used = 0; 7796 bf = NULL; 7797 7798 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7799 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 7800 7801 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7802 7803 for (;;) { 7804 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7805 ifq_set_oactive(ifq); 7806 break; 7807 } 7808 7809 m = ifq_dequeue(ifq); 7810 if (m == NULL) { 7811 break; 7812 } 7813 7814 sqe = sq + idx; 7815 ms = &tx->tx_slots[idx]; 7816 memset(sqe, 0, sizeof(*sqe)); 7817 7818 /* ctrl segment */ 7819 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7820 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7821 /* always generate a completion event */ 7822 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7823 7824 /* eth segment */ 7825 csum = 0; 7826 if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 7827 csum |= MCX_SQE_L3_CSUM; 7828 if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) 7829 csum |= MCX_SQE_L4_CSUM; 7830 sqe->sqe_mss_csum = htobe32(csum); 7831 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7832 #if NVLAN > 0 7833 if (m->m_flags & M_VLANTAG) { 7834 struct ether_vlan_header *evh; 7835 evh = (struct ether_vlan_header *) 7836 &sqe->sqe_inline_headers; 7837 7838 /* slightly cheaper vlan_inject() */ 7839 m_copydata(m, 0, ETHER_HDR_LEN, evh); 7840 evh->evl_proto = evh->evl_encap_proto; 7841 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7842 evh->evl_tag = htons(m->m_pkthdr.ether_vtag); 7843 7844 m_adj(m, ETHER_HDR_LEN); 7845 } else 7846 #endif 7847 { 7848 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7849 sqe->sqe_inline_headers); 7850 m_adj(m, MCX_SQ_INLINE_SIZE); 7851 } 7852 7853 if (mcx_load_mbuf(sc, ms, m) != 0) { 7854 m_freem(m); 7855 ifp->if_oerrors++; 7856 continue; 7857 } 7858 bf = (uint64_t *)sqe; 7859 7860 #if NBPFILTER > 0 7861 if (ifp->if_bpf) 7862 bpf_mtap_hdr(ifp->if_bpf, 7863 (caddr_t)sqe->sqe_inline_headers, 7864 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 7865 #endif 7866 map = ms->ms_map; 7867 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7868 BUS_DMASYNC_PREWRITE); 7869 7870 sqe->sqe_ds_sq_num = 7871 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7872 (map->dm_nsegs + 3)); 7873 7874 /* data segment - first wqe has one segment */ 7875 sqs = sqe->sqe_segs; 7876 seg = 0; 7877 nseg = 1; 7878 for (i = 0; i < map->dm_nsegs; i++) { 7879 if (seg == nseg) { 7880 /* next slot */ 7881 idx++; 7882 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7883 idx = 0; 7884 tx->tx_prod++; 7885 used++; 7886 7887 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7888 seg = 0; 7889 nseg = MCX_SQ_SEGS_PER_SLOT; 7890 } 7891 sqs[seg].sqs_byte_count = 7892 htobe32(map->dm_segs[i].ds_len); 7893 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7894 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7895 seg++; 7896 } 7897 7898 idx++; 7899 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7900 idx = 0; 7901 tx->tx_prod++; 7902 used++; 7903 } 7904 7905 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7906 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 7907 7908 if (used) { 7909 bus_size_t blueflame; 7910 7911 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7912 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 7913 htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell), 7914 tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7915 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7916 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 7917 7918 /* 7919 * write the first 64 bits of the last sqe we produced 7920 * to the blue flame buffer 7921 */ 7922 7923 blueflame = bf_base + tx->tx_bf_offset; 7924 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 7925 blueflame, *bf); 7926 mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE); 7927 7928 /* next write goes to the other buffer */ 7929 tx->tx_bf_offset ^= sc->sc_bf_size; 7930 } 7931 } 7932 7933 static void 7934 mcx_watchdog(struct ifnet *ifp) 7935 { 7936 } 7937 7938 static void 7939 mcx_media_add_types(struct mcx_softc *sc) 7940 { 7941 struct mcx_reg_ptys ptys; 7942 int i; 7943 uint32_t proto_cap; 7944 7945 memset(&ptys, 0, sizeof(ptys)); 7946 ptys.rp_local_port = 1; 7947 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7948 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7949 sizeof(ptys), MCX_CMDQ_SLOT_POLL) != 0) { 7950 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7951 return; 7952 } 7953 7954 proto_cap = betoh32(ptys.rp_eth_proto_cap); 7955 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7956 const struct mcx_eth_proto_capability *cap; 7957 if (!ISSET(proto_cap, 1 << i)) 7958 continue; 7959 7960 cap = &mcx_eth_cap_map[i]; 7961 if (cap->cap_media == 0) 7962 continue; 7963 7964 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 7965 } 7966 } 7967 7968 static void 7969 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 7970 { 7971 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7972 struct mcx_reg_ptys ptys; 7973 int i; 7974 uint32_t proto_oper; 7975 uint64_t media_oper; 7976 7977 memset(&ptys, 0, sizeof(ptys)); 7978 ptys.rp_local_port = 1; 7979 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7980 7981 rw_enter_write(&sc->sc_cmdq_ioctl_lk); 7982 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7983 sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) { 7984 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7985 goto out; 7986 } 7987 7988 proto_oper = betoh32(ptys.rp_eth_proto_oper); 7989 7990 media_oper = 0; 7991 7992 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7993 const struct mcx_eth_proto_capability *cap; 7994 if (!ISSET(proto_oper, 1 << i)) 7995 continue; 7996 7997 cap = &mcx_eth_cap_map[i]; 7998 7999 if (cap->cap_media != 0) 8000 media_oper = cap->cap_media; 8001 } 8002 8003 ifmr->ifm_status = IFM_AVALID; 8004 if (proto_oper != 0) { 8005 ifmr->ifm_status |= IFM_ACTIVE; 8006 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 8007 /* txpause, rxpause, duplex? */ 8008 } 8009 out: 8010 rw_exit_write(&sc->sc_cmdq_ioctl_lk); 8011 } 8012 8013 static int 8014 mcx_media_change(struct ifnet *ifp) 8015 { 8016 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 8017 struct mcx_reg_ptys ptys; 8018 struct mcx_reg_paos paos; 8019 uint32_t media; 8020 int i, error; 8021 8022 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 8023 return EINVAL; 8024 8025 error = 0; 8026 rw_enter_write(&sc->sc_cmdq_ioctl_lk); 8027 8028 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 8029 /* read ptys to get supported media */ 8030 memset(&ptys, 0, sizeof(ptys)); 8031 ptys.rp_local_port = 1; 8032 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8033 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 8034 &ptys, sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) { 8035 printf("%s: unable to read port type/speed\n", 8036 DEVNAME(sc)); 8037 error = EIO; 8038 goto out; 8039 } 8040 8041 media = betoh32(ptys.rp_eth_proto_cap); 8042 } else { 8043 /* map media type */ 8044 media = 0; 8045 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 8046 const struct mcx_eth_proto_capability *cap; 8047 8048 cap = &mcx_eth_cap_map[i]; 8049 if (cap->cap_media == 8050 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 8051 media = (1 << i); 8052 break; 8053 } 8054 } 8055 } 8056 8057 /* disable the port */ 8058 memset(&paos, 0, sizeof(paos)); 8059 paos.rp_local_port = 1; 8060 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 8061 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 8062 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 8063 sizeof(paos), MCX_CMDQ_SLOT_IOCTL) != 0) { 8064 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 8065 error = EIO; 8066 goto out; 8067 } 8068 8069 memset(&ptys, 0, sizeof(ptys)); 8070 ptys.rp_local_port = 1; 8071 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8072 ptys.rp_eth_proto_admin = htobe32(media); 8073 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 8074 sizeof(ptys), MCX_CMDQ_SLOT_IOCTL) != 0) { 8075 printf("%s: unable to set port media type/speed\n", 8076 DEVNAME(sc)); 8077 error = EIO; 8078 /* continue on */ 8079 } 8080 8081 /* re-enable the port to start negotiation */ 8082 memset(&paos, 0, sizeof(paos)); 8083 paos.rp_local_port = 1; 8084 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 8085 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 8086 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 8087 sizeof(paos), MCX_CMDQ_SLOT_IOCTL) != 0) { 8088 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 8089 error = EIO; 8090 } 8091 8092 out: 8093 rw_exit_write(&sc->sc_cmdq_ioctl_lk); 8094 return error; 8095 } 8096 8097 static void 8098 mcx_port_change(void *xsc) 8099 { 8100 struct mcx_softc *sc = xsc; 8101 struct ifnet *ifp = &sc->sc_ac.ac_if; 8102 struct mcx_reg_ptys ptys = { 8103 .rp_local_port = 1, 8104 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 8105 }; 8106 int link_state = LINK_STATE_DOWN; 8107 int slot; 8108 8109 if (cold) { 8110 slot = MCX_CMDQ_SLOT_POLL; 8111 } else 8112 slot = MCX_CMDQ_SLOT_LINK; 8113 8114 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 8115 sizeof(ptys), slot) == 0) { 8116 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 8117 uint64_t baudrate = 0; 8118 unsigned int i; 8119 8120 if (proto_oper != 0) 8121 link_state = LINK_STATE_FULL_DUPLEX; 8122 8123 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 8124 const struct mcx_eth_proto_capability *cap; 8125 if (!ISSET(proto_oper, 1 << i)) 8126 continue; 8127 8128 cap = &mcx_eth_cap_map[i]; 8129 if (cap->cap_baudrate == 0) 8130 continue; 8131 8132 baudrate = cap->cap_baudrate; 8133 break; 8134 } 8135 8136 ifp->if_baudrate = baudrate; 8137 } 8138 8139 if (link_state != ifp->if_link_state) { 8140 ifp->if_link_state = link_state; 8141 if_link_state_change(ifp); 8142 } 8143 } 8144 8145 static inline uint32_t 8146 mcx_rd(struct mcx_softc *sc, bus_size_t r) 8147 { 8148 uint32_t word; 8149 8150 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 8151 8152 return (betoh32(word)); 8153 } 8154 8155 static inline void 8156 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 8157 { 8158 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 8159 } 8160 8161 static inline void 8162 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 8163 { 8164 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 8165 } 8166 8167 static uint64_t 8168 mcx_timer(struct mcx_softc *sc) 8169 { 8170 uint32_t hi, lo, ni; 8171 8172 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8173 for (;;) { 8174 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 8175 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 8176 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8177 8178 if (ni == hi) 8179 break; 8180 8181 hi = ni; 8182 } 8183 8184 return (((uint64_t)hi << 32) | (uint64_t)lo); 8185 } 8186 8187 static int 8188 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 8189 bus_size_t size, u_int align) 8190 { 8191 mxm->mxm_size = size; 8192 8193 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 8194 mxm->mxm_size, 0, 8195 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 8196 &mxm->mxm_map) != 0) 8197 return (1); 8198 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 8199 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 8200 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 8201 goto destroy; 8202 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 8203 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 8204 goto free; 8205 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 8206 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 8207 goto unmap; 8208 8209 return (0); 8210 unmap: 8211 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8212 free: 8213 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8214 destroy: 8215 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8216 return (1); 8217 } 8218 8219 static void 8220 mcx_dmamem_zero(struct mcx_dmamem *mxm) 8221 { 8222 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 8223 } 8224 8225 static void 8226 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 8227 { 8228 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 8229 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8230 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8231 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8232 } 8233 8234 static int 8235 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 8236 { 8237 bus_dma_segment_t *segs; 8238 bus_size_t len = pages * MCX_PAGE_SIZE; 8239 size_t seglen; 8240 8241 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 8242 if (segs == NULL) 8243 return (-1); 8244 8245 seglen = sizeof(*segs) * pages; 8246 8247 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 8248 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 8249 goto free_segs; 8250 8251 if (mhm->mhm_seg_count < pages) { 8252 size_t nseglen; 8253 8254 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 8255 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 8256 if (mhm->mhm_segs == NULL) 8257 goto free_dmamem; 8258 8259 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 8260 8261 memcpy(mhm->mhm_segs, segs, nseglen); 8262 8263 free(segs, M_DEVBUF, seglen); 8264 8265 segs = mhm->mhm_segs; 8266 seglen = nseglen; 8267 } else 8268 mhm->mhm_segs = segs; 8269 8270 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 8271 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 8272 &mhm->mhm_map) != 0) 8273 goto free_dmamem; 8274 8275 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 8276 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 8277 goto destroy; 8278 8279 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8280 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 8281 8282 mhm->mhm_npages = pages; 8283 8284 return (0); 8285 8286 destroy: 8287 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8288 free_dmamem: 8289 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8290 free_segs: 8291 free(segs, M_DEVBUF, seglen); 8292 mhm->mhm_segs = NULL; 8293 8294 return (-1); 8295 } 8296 8297 static void 8298 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 8299 { 8300 if (mhm->mhm_npages == 0) 8301 return; 8302 8303 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8304 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 8305 8306 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 8307 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8308 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8309 free(mhm->mhm_segs, M_DEVBUF, 8310 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 8311 8312 mhm->mhm_npages = 0; 8313 } 8314 8315 #if NKSTAT > 0 8316 struct mcx_ppcnt { 8317 char name[KSTAT_KV_NAMELEN]; 8318 enum kstat_kv_unit unit; 8319 }; 8320 8321 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 8322 { "Good Tx", KSTAT_KV_U_PACKETS, }, 8323 { "Good Rx", KSTAT_KV_U_PACKETS, }, 8324 { "FCS errs", KSTAT_KV_U_PACKETS, }, 8325 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8326 { "Good Tx", KSTAT_KV_U_BYTES, }, 8327 { "Good Rx", KSTAT_KV_U_BYTES, }, 8328 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 8329 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 8330 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 8331 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 8332 { "In Range Len", KSTAT_KV_U_PACKETS, }, 8333 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 8334 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 8335 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 8336 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 8337 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 8338 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 8339 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8340 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8341 }; 8342 CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 8343 8344 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 8345 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 8346 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 8347 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 8348 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 8349 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 8350 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 8351 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 8352 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 8353 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 8354 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 8355 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 8356 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 8357 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 8358 }; 8359 CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 8360 8361 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 8362 { "Drop Events", KSTAT_KV_U_PACKETS, }, 8363 { "Octets", KSTAT_KV_U_BYTES, }, 8364 { "Packets", KSTAT_KV_U_PACKETS, }, 8365 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 8366 { "Multicasts", KSTAT_KV_U_PACKETS, }, 8367 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 8368 { "Undersize", KSTAT_KV_U_PACKETS, }, 8369 { "Oversize", KSTAT_KV_U_PACKETS, }, 8370 { "Fragments", KSTAT_KV_U_PACKETS, }, 8371 { "Jabbers", KSTAT_KV_U_PACKETS, }, 8372 { "Collisions", KSTAT_KV_U_NONE, }, 8373 { "64B", KSTAT_KV_U_PACKETS, }, 8374 { "65-127B", KSTAT_KV_U_PACKETS, }, 8375 { "128-255B", KSTAT_KV_U_PACKETS, }, 8376 { "256-511B", KSTAT_KV_U_PACKETS, }, 8377 { "512-1023B", KSTAT_KV_U_PACKETS, }, 8378 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 8379 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 8380 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 8381 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 8382 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 8383 }; 8384 CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 8385 8386 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 8387 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8388 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 8389 { "Single Colls", KSTAT_KV_U_PACKETS, }, 8390 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 8391 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 8392 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 8393 { "Late Colls", KSTAT_KV_U_NONE, }, 8394 { "Exess Colls", KSTAT_KV_U_NONE, }, 8395 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 8396 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 8397 { "Too Long", KSTAT_KV_U_PACKETS, }, 8398 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 8399 { "Symbol Errs", KSTAT_KV_U_NONE, }, 8400 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 8401 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8402 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8403 }; 8404 CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 8405 8406 struct mcx_kstat_ppcnt { 8407 const char *ksp_name; 8408 const struct mcx_ppcnt *ksp_tpl; 8409 unsigned int ksp_n; 8410 uint8_t ksp_grp; 8411 }; 8412 8413 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 8414 .ksp_name = "ieee802.3", 8415 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 8416 .ksp_n = nitems(mcx_ppcnt_ieee8023_tpl), 8417 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 8418 }; 8419 8420 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 8421 .ksp_name = "rfc2863", 8422 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 8423 .ksp_n = nitems(mcx_ppcnt_rfc2863_tpl), 8424 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 8425 }; 8426 8427 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 8428 .ksp_name = "rfc2819", 8429 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 8430 .ksp_n = nitems(mcx_ppcnt_rfc2819_tpl), 8431 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 8432 }; 8433 8434 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 8435 .ksp_name = "rfc3635", 8436 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 8437 .ksp_n = nitems(mcx_ppcnt_rfc3635_tpl), 8438 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 8439 }; 8440 8441 static int mcx_kstat_ppcnt_read(struct kstat *); 8442 8443 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 8444 static void mcx_kstat_attach_queues(struct mcx_softc *sc); 8445 8446 static struct kstat * 8447 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 8448 const struct mcx_kstat_ppcnt *ksp) 8449 { 8450 struct kstat *ks; 8451 struct kstat_kv *kvs; 8452 unsigned int i; 8453 8454 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 8455 if (ks == NULL) 8456 return (NULL); 8457 8458 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 8459 M_DEVBUF, M_WAITOK); 8460 8461 for (i = 0; i < ksp->ksp_n; i++) { 8462 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 8463 8464 kstat_kv_unit_init(&kvs[i], tpl->name, 8465 KSTAT_KV_T_COUNTER64, tpl->unit); 8466 } 8467 8468 ks->ks_softc = sc; 8469 ks->ks_ptr = (void *)ksp; 8470 ks->ks_data = kvs; 8471 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 8472 ks->ks_read = mcx_kstat_ppcnt_read; 8473 kstat_set_wlock(ks, &sc->sc_cmdq_kstat_lk); 8474 8475 kstat_install(ks); 8476 8477 return (ks); 8478 } 8479 8480 static void 8481 mcx_kstat_attach(struct mcx_softc *sc) 8482 { 8483 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 8484 &mcx_kstat_ppcnt_ieee8023); 8485 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 8486 &mcx_kstat_ppcnt_rfc2863); 8487 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 8488 &mcx_kstat_ppcnt_rfc2819); 8489 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 8490 &mcx_kstat_ppcnt_rfc3635); 8491 8492 mcx_kstat_attach_tmps(sc); 8493 mcx_kstat_attach_queues(sc); 8494 } 8495 8496 static int 8497 mcx_kstat_ppcnt_read(struct kstat *ks) 8498 { 8499 struct mcx_softc *sc = ks->ks_softc; 8500 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 8501 struct mcx_reg_ppcnt ppcnt = { 8502 .ppcnt_grp = ksp->ksp_grp, 8503 .ppcnt_local_port = 1, 8504 }; 8505 struct kstat_kv *kvs = ks->ks_data; 8506 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 8507 unsigned int i; 8508 int rv; 8509 8510 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 8511 &ppcnt, sizeof(ppcnt), MCX_CMDQ_SLOT_KSTAT); 8512 if (rv != 0) 8513 return (EIO); 8514 8515 nanouptime(&ks->ks_updated); 8516 8517 for (i = 0; i < ksp->ksp_n; i++) 8518 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 8519 8520 return (0); 8521 } 8522 8523 struct mcx_kstat_mtmp { 8524 struct kstat_kv ktmp_name; 8525 struct kstat_kv ktmp_temperature; 8526 struct kstat_kv ktmp_threshold_lo; 8527 struct kstat_kv ktmp_threshold_hi; 8528 }; 8529 8530 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 8531 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 8532 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 8533 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 8534 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 8535 }; 8536 8537 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 8538 8539 static int mcx_kstat_mtmp_read(struct kstat *); 8540 8541 static void 8542 mcx_kstat_attach_tmps(struct mcx_softc *sc) 8543 { 8544 struct kstat *ks; 8545 struct mcx_reg_mcam mcam; 8546 struct mcx_reg_mtcap mtcap; 8547 struct mcx_kstat_mtmp *ktmp; 8548 uint64_t map; 8549 unsigned int i, n; 8550 8551 memset(&mtcap, 0, sizeof(mtcap)); 8552 memset(&mcam, 0, sizeof(mcam)); 8553 8554 if (sc->sc_mcam_reg == 0) { 8555 /* no management capabilities */ 8556 return; 8557 } 8558 8559 if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ, 8560 &mcam, sizeof(mcam), MCX_CMDQ_SLOT_POLL) != 0) { 8561 /* unable to check management capabilities? */ 8562 return; 8563 } 8564 8565 if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask, 8566 MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) { 8567 /* no sensor map */ 8568 return; 8569 } 8570 8571 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8572 &mtcap, sizeof(mtcap), MCX_CMDQ_SLOT_POLL) != 0) { 8573 /* unable to find temperature sensors */ 8574 return; 8575 } 8576 8577 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8578 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8579 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8580 8581 n = 0; 8582 map = bemtoh64(&mtcap.mtcap_sensor_map); 8583 for (i = 0; i < sizeof(map) * NBBY; i++) { 8584 if (!ISSET(map, (1ULL << i))) 8585 continue; 8586 8587 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8588 KSTAT_T_KV, 0); 8589 if (ks == NULL) { 8590 /* unable to attach temperature sensor %u, i */ 8591 continue; 8592 } 8593 8594 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8595 *ktmp = mcx_kstat_mtmp_tpl; 8596 8597 ks->ks_data = ktmp; 8598 ks->ks_datalen = sizeof(*ktmp); 8599 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8600 ks->ks_read = mcx_kstat_mtmp_read; 8601 kstat_set_wlock(ks, &sc->sc_cmdq_kstat_lk); 8602 8603 ks->ks_softc = sc; 8604 kstat_install(ks); 8605 8606 sc->sc_kstat_mtmp[n++] = ks; 8607 if (n >= sc->sc_kstat_mtmp_count) 8608 break; 8609 } 8610 } 8611 8612 static uint64_t 8613 mcx_tmp_to_uK(uint16_t *t) 8614 { 8615 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8616 mt *= 1000000 / 8; /* convert to uC */ 8617 mt += 273150000; /* convert to uK */ 8618 8619 return (mt); 8620 } 8621 8622 static int 8623 mcx_kstat_mtmp_read(struct kstat *ks) 8624 { 8625 struct mcx_softc *sc = ks->ks_softc; 8626 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8627 struct mcx_reg_mtmp mtmp; 8628 int rv; 8629 struct timeval updated; 8630 8631 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8632 8633 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8634 return (0); 8635 8636 memset(&mtmp, 0, sizeof(mtmp)); 8637 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8638 8639 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8640 &mtmp, sizeof(mtmp), MCX_CMDQ_SLOT_KSTAT); 8641 if (rv != 0) 8642 return (EIO); 8643 8644 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8645 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8646 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8647 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8648 kstat_kv_temp(&ktmp->ktmp_temperature) = 8649 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8650 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8651 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8652 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8653 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8654 8655 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8656 8657 return (0); 8658 } 8659 8660 struct mcx_queuestat { 8661 char name[KSTAT_KV_NAMELEN]; 8662 enum kstat_kv_type type; 8663 }; 8664 8665 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = { 8666 { "RQ SW prod", KSTAT_KV_T_COUNTER64 }, 8667 { "RQ HW prod", KSTAT_KV_T_COUNTER64 }, 8668 { "RQ HW cons", KSTAT_KV_T_COUNTER64 }, 8669 { "RQ HW state", KSTAT_KV_T_ISTR }, 8670 8671 { "SQ SW prod", KSTAT_KV_T_COUNTER64 }, 8672 { "SQ SW cons", KSTAT_KV_T_COUNTER64 }, 8673 { "SQ HW prod", KSTAT_KV_T_COUNTER64 }, 8674 { "SQ HW cons", KSTAT_KV_T_COUNTER64 }, 8675 { "SQ HW state", KSTAT_KV_T_ISTR }, 8676 8677 { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, 8678 { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, 8679 { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, 8680 { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, 8681 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, 8682 { "CQ HW status", KSTAT_KV_T_ISTR }, 8683 { "CQ HW state", KSTAT_KV_T_ISTR }, 8684 8685 { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, 8686 { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, 8687 { "EQ HW cons", KSTAT_KV_T_COUNTER64 }, 8688 { "EQ HW status", KSTAT_KV_T_ISTR }, 8689 { "EQ HW state", KSTAT_KV_T_ISTR }, 8690 }; 8691 8692 static int mcx_kstat_queue_read(struct kstat *); 8693 8694 static void 8695 mcx_kstat_attach_queues(struct mcx_softc *sc) 8696 { 8697 struct kstat *ks; 8698 struct kstat_kv *kvs; 8699 int q, i; 8700 8701 for (q = 0; q < intrmap_count(sc->sc_intrmap); q++) { 8702 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, 8703 KSTAT_T_KV, 0); 8704 if (ks == NULL) { 8705 /* unable to attach queue stats %u, q */ 8706 continue; 8707 } 8708 8709 kvs = mallocarray(nitems(mcx_queue_kstat_tpl), 8710 sizeof(*kvs), M_DEVBUF, M_WAITOK); 8711 8712 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) { 8713 const struct mcx_queuestat *tpl = 8714 &mcx_queue_kstat_tpl[i]; 8715 8716 kstat_kv_init(&kvs[i], tpl->name, tpl->type); 8717 } 8718 8719 ks->ks_softc = &sc->sc_queues[q]; 8720 ks->ks_data = kvs; 8721 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs); 8722 ks->ks_read = mcx_kstat_queue_read; 8723 8724 sc->sc_queues[q].q_kstat = ks; 8725 kstat_install(ks); 8726 } 8727 } 8728 8729 static int 8730 mcx_kstat_queue_read(struct kstat *ks) 8731 { 8732 struct mcx_queues *q = ks->ks_softc; 8733 struct mcx_softc *sc = q->q_sc; 8734 struct kstat_kv *kvs = ks->ks_data; 8735 union { 8736 struct mcx_rq_ctx rq; 8737 struct mcx_sq_ctx sq; 8738 struct mcx_cq_ctx cq; 8739 struct mcx_eq_ctx eq; 8740 } u; 8741 const char *text; 8742 int error = 0; 8743 8744 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) { 8745 error = EIO; 8746 goto out; 8747 } 8748 8749 kstat_kv_u64(kvs++) = q->q_rx.rx_prod; 8750 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter); 8751 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter); 8752 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >> 8753 MCX_RQ_CTX_STATE_SHIFT) { 8754 case MCX_RQ_CTX_STATE_RST: 8755 text = "RST"; 8756 break; 8757 case MCX_RQ_CTX_STATE_RDY: 8758 text = "RDY"; 8759 break; 8760 case MCX_RQ_CTX_STATE_ERR: 8761 text = "ERR"; 8762 break; 8763 default: 8764 text = "unknown"; 8765 break; 8766 } 8767 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8768 kvs++; 8769 8770 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) { 8771 error = EIO; 8772 goto out; 8773 } 8774 8775 kstat_kv_u64(kvs++) = q->q_tx.tx_prod; 8776 kstat_kv_u64(kvs++) = q->q_tx.tx_cons; 8777 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter); 8778 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter); 8779 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >> 8780 MCX_SQ_CTX_STATE_SHIFT) { 8781 case MCX_SQ_CTX_STATE_RST: 8782 text = "RST"; 8783 break; 8784 case MCX_SQ_CTX_STATE_RDY: 8785 text = "RDY"; 8786 break; 8787 case MCX_SQ_CTX_STATE_ERR: 8788 text = "ERR"; 8789 break; 8790 default: 8791 text = "unknown"; 8792 break; 8793 } 8794 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8795 kvs++; 8796 8797 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) { 8798 error = EIO; 8799 goto out; 8800 } 8801 8802 kstat_kv_u64(kvs++) = q->q_cq.cq_cons; 8803 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); 8804 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); 8805 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); 8806 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); 8807 8808 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >> 8809 MCX_CQ_CTX_STATUS_SHIFT) { 8810 case MCX_CQ_CTX_STATUS_OK: 8811 text = "OK"; 8812 break; 8813 case MCX_CQ_CTX_STATUS_OVERFLOW: 8814 text = "overflow"; 8815 break; 8816 case MCX_CQ_CTX_STATUS_WRITE_FAIL: 8817 text = "write fail"; 8818 break; 8819 default: 8820 text = "unknown"; 8821 break; 8822 } 8823 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8824 kvs++; 8825 8826 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >> 8827 MCX_CQ_CTX_STATE_SHIFT) { 8828 case MCX_CQ_CTX_STATE_SOLICITED: 8829 text = "solicited"; 8830 break; 8831 case MCX_CQ_CTX_STATE_ARMED: 8832 text = "armed"; 8833 break; 8834 case MCX_CQ_CTX_STATE_FIRED: 8835 text = "fired"; 8836 break; 8837 default: 8838 text = "unknown"; 8839 break; 8840 } 8841 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8842 kvs++; 8843 8844 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { 8845 error = EIO; 8846 goto out; 8847 } 8848 8849 kstat_kv_u64(kvs++) = q->q_eq.eq_cons; 8850 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter); 8851 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter); 8852 8853 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >> 8854 MCX_EQ_CTX_STATUS_SHIFT) { 8855 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE: 8856 text = "write fail"; 8857 break; 8858 case MCX_EQ_CTX_STATUS_OK: 8859 text = "OK"; 8860 break; 8861 default: 8862 text = "unknown"; 8863 break; 8864 } 8865 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8866 kvs++; 8867 8868 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >> 8869 MCX_EQ_CTX_STATE_SHIFT) { 8870 case MCX_EQ_CTX_STATE_ARMED: 8871 text = "armed"; 8872 break; 8873 case MCX_EQ_CTX_STATE_FIRED: 8874 text = "fired"; 8875 break; 8876 default: 8877 text = "unknown"; 8878 break; 8879 } 8880 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8881 kvs++; 8882 8883 nanouptime(&ks->ks_updated); 8884 out: 8885 return (error); 8886 } 8887 8888 #endif /* NKSTAT > 0 */ 8889 8890 static unsigned int 8891 mcx_timecounter_read(struct timecounter *tc) 8892 { 8893 struct mcx_softc *sc = tc->tc_priv; 8894 8895 return (mcx_rd(sc, MCX_INTERNAL_TIMER_L)); 8896 } 8897 8898 static void 8899 mcx_timecounter_attach(struct mcx_softc *sc) 8900 { 8901 struct timecounter *tc = &sc->sc_timecounter; 8902 8903 tc->tc_get_timecount = mcx_timecounter_read; 8904 tc->tc_counter_mask = ~0U; 8905 tc->tc_frequency = sc->sc_khz * 1000; 8906 tc->tc_name = sc->sc_dev.dv_xname; 8907 tc->tc_quality = -100; 8908 tc->tc_priv = sc; 8909 8910 tc_init(tc); 8911 } 8912