1 /* $OpenBSD: if_mcx.c,v 1.93 2021/01/04 23:12:05 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 #include "vlan.h" 22 #include "kstat.h" 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/kernel.h> 29 #include <sys/socket.h> 30 #include <sys/device.h> 31 #include <sys/pool.h> 32 #include <sys/queue.h> 33 #include <sys/timeout.h> 34 #include <sys/task.h> 35 #include <sys/atomic.h> 36 #include <sys/timetc.h> 37 38 #include <machine/bus.h> 39 #include <machine/intr.h> 40 41 #include <net/if.h> 42 #include <net/if_dl.h> 43 #include <net/if_media.h> 44 #include <net/toeplitz.h> 45 46 #if NBPFILTER > 0 47 #include <net/bpf.h> 48 #endif 49 50 #if NKSTAT > 0 51 #include <sys/kstat.h> 52 #endif 53 54 #include <netinet/in.h> 55 #include <netinet/if_ether.h> 56 57 #include <dev/pci/pcireg.h> 58 #include <dev/pci/pcivar.h> 59 #include <dev/pci/pcidevs.h> 60 61 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 62 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 63 64 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 65 66 #define MCX_FW_VER 0x0000 67 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 68 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 69 #define MCX_CMDIF_FW_SUBVER 0x0004 70 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 71 #define MCX_CMDIF(_v) ((_v) >> 16) 72 73 #define MCX_ISSI 1 /* as per the PRM */ 74 #define MCX_CMD_IF_SUPPORTED 5 75 76 #define MCX_HARDMTU 9500 77 78 #define MCX_PAGE_SHIFT 12 79 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 80 81 /* queue sizes */ 82 #define MCX_LOG_EQ_SIZE 7 83 #define MCX_LOG_CQ_SIZE 12 84 #define MCX_LOG_RQ_SIZE 10 85 #define MCX_LOG_SQ_SIZE 11 86 87 #define MCX_MAX_QUEUES 1 88 89 /* completion event moderation - about 10khz, or 90% of the cq */ 90 #define MCX_CQ_MOD_PERIOD 50 91 #define MCX_CQ_MOD_COUNTER \ 92 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 93 94 #define MCX_LOG_SQ_ENTRY_SIZE 6 95 #define MCX_SQ_ENTRY_MAX_SLOTS 4 96 #define MCX_SQ_SEGS_PER_SLOT \ 97 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 98 #define MCX_SQ_MAX_SEGMENTS \ 99 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 100 101 #define MCX_LOG_FLOW_TABLE_SIZE 5 102 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 103 #define MCX_NUM_MCAST_FLOWS \ 104 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 105 106 #define MCX_SQ_INLINE_SIZE 18 107 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 108 109 /* doorbell offsets */ 110 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 111 112 #define MCX_CQ_DOORBELL_BASE 0 113 #define MCX_CQ_DOORBELL_STRIDE 64 114 115 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 116 #define MCX_WQ_DOORBELL_STRIDE 64 117 /* make sure the doorbells fit */ 118 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 119 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 120 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 121 122 #define MCX_WQ_DOORBELL_MASK 0xffff 123 124 /* uar registers */ 125 #define MCX_UAR_CQ_DOORBELL 0x20 126 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 127 #define MCX_UAR_EQ_DOORBELL 0x48 128 #define MCX_UAR_BF 0x800 129 130 #define MCX_CMDQ_ADDR_HI 0x0010 131 #define MCX_CMDQ_ADDR_LO 0x0014 132 #define MCX_CMDQ_ADDR_NMASK 0xfff 133 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 134 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 135 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 136 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 137 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 138 139 #define MCX_CMDQ_DOORBELL 0x0018 140 141 #define MCX_STATE 0x01fc 142 #define MCX_STATE_MASK (1 << 31) 143 #define MCX_STATE_INITIALIZING (1 << 31) 144 #define MCX_STATE_READY (0 << 31) 145 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 146 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 147 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 148 149 #define MCX_INTERNAL_TIMER 0x1000 150 #define MCX_INTERNAL_TIMER_H 0x1000 151 #define MCX_INTERNAL_TIMER_L 0x1004 152 153 #define MCX_CLEAR_INT 0x100c 154 155 #define MCX_REG_OP_WRITE 0 156 #define MCX_REG_OP_READ 1 157 158 #define MCX_REG_PMLP 0x5002 159 #define MCX_REG_PMTU 0x5003 160 #define MCX_REG_PTYS 0x5004 161 #define MCX_REG_PAOS 0x5006 162 #define MCX_REG_PFCC 0x5007 163 #define MCX_REG_PPCNT 0x5008 164 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 165 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 166 #define MCX_REG_MCIA 0x9014 167 168 #define MCX_ETHER_CAP_SGMII 0 169 #define MCX_ETHER_CAP_1000_KX 1 170 #define MCX_ETHER_CAP_10G_CX4 2 171 #define MCX_ETHER_CAP_10G_KX4 3 172 #define MCX_ETHER_CAP_10G_KR 4 173 #define MCX_ETHER_CAP_40G_CR4 6 174 #define MCX_ETHER_CAP_40G_KR4 7 175 #define MCX_ETHER_CAP_10G_CR 12 176 #define MCX_ETHER_CAP_10G_SR 13 177 #define MCX_ETHER_CAP_10G_LR 14 178 #define MCX_ETHER_CAP_40G_SR4 15 179 #define MCX_ETHER_CAP_40G_LR4 16 180 #define MCX_ETHER_CAP_50G_SR2 18 181 #define MCX_ETHER_CAP_100G_CR4 20 182 #define MCX_ETHER_CAP_100G_SR4 21 183 #define MCX_ETHER_CAP_100G_KR4 22 184 #define MCX_ETHER_CAP_25G_CR 27 185 #define MCX_ETHER_CAP_25G_KR 28 186 #define MCX_ETHER_CAP_25G_SR 29 187 #define MCX_ETHER_CAP_50G_CR2 30 188 #define MCX_ETHER_CAP_50G_KR2 31 189 190 #define MCX_MAX_CQE 32 191 192 #define MCX_CMD_QUERY_HCA_CAP 0x100 193 #define MCX_CMD_QUERY_ADAPTER 0x101 194 #define MCX_CMD_INIT_HCA 0x102 195 #define MCX_CMD_TEARDOWN_HCA 0x103 196 #define MCX_CMD_ENABLE_HCA 0x104 197 #define MCX_CMD_DISABLE_HCA 0x105 198 #define MCX_CMD_QUERY_PAGES 0x107 199 #define MCX_CMD_MANAGE_PAGES 0x108 200 #define MCX_CMD_SET_HCA_CAP 0x109 201 #define MCX_CMD_QUERY_ISSI 0x10a 202 #define MCX_CMD_SET_ISSI 0x10b 203 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 204 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 205 #define MCX_CMD_CREATE_EQ 0x301 206 #define MCX_CMD_DESTROY_EQ 0x302 207 #define MCX_CMD_QUERY_EQ 0x303 208 #define MCX_CMD_CREATE_CQ 0x400 209 #define MCX_CMD_DESTROY_CQ 0x401 210 #define MCX_CMD_QUERY_CQ 0x402 211 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 212 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 213 0x755 214 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 215 #define MCX_CMD_ALLOC_PD 0x800 216 #define MCX_CMD_ALLOC_UAR 0x802 217 #define MCX_CMD_ACCESS_REG 0x805 218 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 219 #define MCX_CMD_CREATE_TIR 0x900 220 #define MCX_CMD_DESTROY_TIR 0x902 221 #define MCX_CMD_CREATE_SQ 0x904 222 #define MCX_CMD_MODIFY_SQ 0x905 223 #define MCX_CMD_DESTROY_SQ 0x906 224 #define MCX_CMD_QUERY_SQ 0x907 225 #define MCX_CMD_CREATE_RQ 0x908 226 #define MCX_CMD_MODIFY_RQ 0x909 227 #define MCX_CMD_DESTROY_RQ 0x90a 228 #define MCX_CMD_QUERY_RQ 0x90b 229 #define MCX_CMD_CREATE_TIS 0x912 230 #define MCX_CMD_DESTROY_TIS 0x914 231 #define MCX_CMD_CREATE_RQT 0x916 232 #define MCX_CMD_DESTROY_RQT 0x918 233 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 234 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 235 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 236 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 237 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 238 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 239 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 240 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 241 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 242 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 243 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 244 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 245 246 #define MCX_QUEUE_STATE_RST 0 247 #define MCX_QUEUE_STATE_RDY 1 248 #define MCX_QUEUE_STATE_ERR 3 249 250 #define MCX_FLOW_TABLE_TYPE_RX 0 251 #define MCX_FLOW_TABLE_TYPE_TX 1 252 253 #define MCX_CMDQ_INLINE_DATASIZE 16 254 255 struct mcx_cmdq_entry { 256 uint8_t cq_type; 257 #define MCX_CMDQ_TYPE_PCIE 0x7 258 uint8_t cq_reserved0[3]; 259 260 uint32_t cq_input_length; 261 uint64_t cq_input_ptr; 262 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 263 264 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 265 uint64_t cq_output_ptr; 266 uint32_t cq_output_length; 267 268 uint8_t cq_token; 269 uint8_t cq_signature; 270 uint8_t cq_reserved1[1]; 271 uint8_t cq_status; 272 #define MCX_CQ_STATUS_SHIFT 1 273 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 274 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 275 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 276 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 277 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 278 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 279 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 280 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 281 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 282 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 283 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 284 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 285 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 286 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 287 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 288 (0x10 << MCX_CQ_STATUS_SHIFT) 289 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 290 #define MCX_CQ_STATUS_OWN_MASK 0x1 291 #define MCX_CQ_STATUS_OWN_SW 0x0 292 #define MCX_CQ_STATUS_OWN_HW 0x1 293 } __packed __aligned(8); 294 295 #define MCX_CMDQ_MAILBOX_DATASIZE 512 296 297 struct mcx_cmdq_mailbox { 298 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 299 uint8_t mb_reserved0[48]; 300 uint64_t mb_next_ptr; 301 uint32_t mb_block_number; 302 uint8_t mb_reserved1[1]; 303 uint8_t mb_token; 304 uint8_t mb_ctrl_signature; 305 uint8_t mb_signature; 306 } __packed __aligned(8); 307 308 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 309 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 310 MCX_CMDQ_MAILBOX_ALIGN) 311 /* 312 * command mailbox structres 313 */ 314 315 struct mcx_cmd_enable_hca_in { 316 uint16_t cmd_opcode; 317 uint8_t cmd_reserved0[4]; 318 uint16_t cmd_op_mod; 319 uint8_t cmd_reserved1[2]; 320 uint16_t cmd_function_id; 321 uint8_t cmd_reserved2[4]; 322 } __packed __aligned(4); 323 324 struct mcx_cmd_enable_hca_out { 325 uint8_t cmd_status; 326 uint8_t cmd_reserved0[3]; 327 uint32_t cmd_syndrome; 328 uint8_t cmd_reserved1[4]; 329 } __packed __aligned(4); 330 331 struct mcx_cmd_init_hca_in { 332 uint16_t cmd_opcode; 333 uint8_t cmd_reserved0[4]; 334 uint16_t cmd_op_mod; 335 uint8_t cmd_reserved1[8]; 336 } __packed __aligned(4); 337 338 struct mcx_cmd_init_hca_out { 339 uint8_t cmd_status; 340 uint8_t cmd_reserved0[3]; 341 uint32_t cmd_syndrome; 342 uint8_t cmd_reserved1[8]; 343 } __packed __aligned(4); 344 345 struct mcx_cmd_teardown_hca_in { 346 uint16_t cmd_opcode; 347 uint8_t cmd_reserved0[4]; 348 uint16_t cmd_op_mod; 349 uint8_t cmd_reserved1[2]; 350 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 351 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 352 uint16_t cmd_profile; 353 uint8_t cmd_reserved2[4]; 354 } __packed __aligned(4); 355 356 struct mcx_cmd_teardown_hca_out { 357 uint8_t cmd_status; 358 uint8_t cmd_reserved0[3]; 359 uint32_t cmd_syndrome; 360 uint8_t cmd_reserved1[8]; 361 } __packed __aligned(4); 362 363 struct mcx_cmd_access_reg_in { 364 uint16_t cmd_opcode; 365 uint8_t cmd_reserved0[4]; 366 uint16_t cmd_op_mod; 367 uint8_t cmd_reserved1[2]; 368 uint16_t cmd_register_id; 369 uint32_t cmd_argument; 370 } __packed __aligned(4); 371 372 struct mcx_cmd_access_reg_out { 373 uint8_t cmd_status; 374 uint8_t cmd_reserved0[3]; 375 uint32_t cmd_syndrome; 376 uint8_t cmd_reserved1[8]; 377 } __packed __aligned(4); 378 379 struct mcx_reg_pmtu { 380 uint8_t rp_reserved1; 381 uint8_t rp_local_port; 382 uint8_t rp_reserved2[2]; 383 uint16_t rp_max_mtu; 384 uint8_t rp_reserved3[2]; 385 uint16_t rp_admin_mtu; 386 uint8_t rp_reserved4[2]; 387 uint16_t rp_oper_mtu; 388 uint8_t rp_reserved5[2]; 389 } __packed __aligned(4); 390 391 struct mcx_reg_ptys { 392 uint8_t rp_reserved1; 393 uint8_t rp_local_port; 394 uint8_t rp_reserved2; 395 uint8_t rp_proto_mask; 396 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 397 uint8_t rp_reserved3[8]; 398 uint32_t rp_eth_proto_cap; 399 uint8_t rp_reserved4[8]; 400 uint32_t rp_eth_proto_admin; 401 uint8_t rp_reserved5[8]; 402 uint32_t rp_eth_proto_oper; 403 uint8_t rp_reserved6[24]; 404 } __packed __aligned(4); 405 406 struct mcx_reg_paos { 407 uint8_t rp_reserved1; 408 uint8_t rp_local_port; 409 uint8_t rp_admin_status; 410 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 411 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 412 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 413 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 414 uint8_t rp_oper_status; 415 #define MCX_REG_PAOS_OPER_STATUS_UP 1 416 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 417 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 418 uint8_t rp_admin_state_update; 419 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 420 uint8_t rp_reserved2[11]; 421 } __packed __aligned(4); 422 423 struct mcx_reg_pfcc { 424 uint8_t rp_reserved1; 425 uint8_t rp_local_port; 426 uint8_t rp_reserved2[3]; 427 uint8_t rp_prio_mask_tx; 428 uint8_t rp_reserved3; 429 uint8_t rp_prio_mask_rx; 430 uint8_t rp_pptx_aptx; 431 uint8_t rp_pfctx; 432 uint8_t rp_fctx_dis; 433 uint8_t rp_reserved4; 434 uint8_t rp_pprx_aprx; 435 uint8_t rp_pfcrx; 436 uint8_t rp_reserved5[2]; 437 uint16_t rp_dev_stall_min; 438 uint16_t rp_dev_stall_crit; 439 uint8_t rp_reserved6[12]; 440 } __packed __aligned(4); 441 442 #define MCX_PMLP_MODULE_NUM_MASK 0xff 443 struct mcx_reg_pmlp { 444 uint8_t rp_rxtx; 445 uint8_t rp_local_port; 446 uint8_t rp_reserved0; 447 uint8_t rp_width; 448 uint32_t rp_lane0_mapping; 449 uint32_t rp_lane1_mapping; 450 uint32_t rp_lane2_mapping; 451 uint32_t rp_lane3_mapping; 452 uint8_t rp_reserved1[44]; 453 } __packed __aligned(4); 454 455 struct mcx_reg_ppcnt { 456 uint8_t ppcnt_swid; 457 uint8_t ppcnt_local_port; 458 uint8_t ppcnt_pnat; 459 uint8_t ppcnt_grp; 460 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 461 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 462 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 463 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 464 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 465 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 466 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 467 468 uint8_t ppcnt_clr; 469 uint8_t ppcnt_reserved1[2]; 470 uint8_t ppcnt_prio_tc; 471 #define MCX_REG_PPCNT_CLR (1 << 7) 472 473 uint8_t ppcnt_counter_set[248]; 474 } __packed __aligned(8); 475 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 476 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 477 sizeof(uint64_t)) == 0); 478 479 enum mcx_ppcnt_ieee8023 { 480 frames_transmitted_ok, 481 frames_received_ok, 482 frame_check_sequence_errors, 483 alignment_errors, 484 octets_transmitted_ok, 485 octets_received_ok, 486 multicast_frames_xmitted_ok, 487 broadcast_frames_xmitted_ok, 488 multicast_frames_received_ok, 489 broadcast_frames_received_ok, 490 in_range_length_errors, 491 out_of_range_length_field, 492 frame_too_long_errors, 493 symbol_error_during_carrier, 494 mac_control_frames_transmitted, 495 mac_control_frames_received, 496 unsupported_opcodes_received, 497 pause_mac_ctrl_frames_received, 498 pause_mac_ctrl_frames_transmitted, 499 500 mcx_ppcnt_ieee8023_count 501 }; 502 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 503 504 enum mcx_ppcnt_rfc2863 { 505 in_octets, 506 in_ucast_pkts, 507 in_discards, 508 in_errors, 509 in_unknown_protos, 510 out_octets, 511 out_ucast_pkts, 512 out_discards, 513 out_errors, 514 in_multicast_pkts, 515 in_broadcast_pkts, 516 out_multicast_pkts, 517 out_broadcast_pkts, 518 519 mcx_ppcnt_rfc2863_count 520 }; 521 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 522 523 enum mcx_ppcnt_rfc2819 { 524 drop_events, 525 octets, 526 pkts, 527 broadcast_pkts, 528 multicast_pkts, 529 crc_align_errors, 530 undersize_pkts, 531 oversize_pkts, 532 fragments, 533 jabbers, 534 collisions, 535 pkts64octets, 536 pkts65to127octets, 537 pkts128to255octets, 538 pkts256to511octets, 539 pkts512to1023octets, 540 pkts1024to1518octets, 541 pkts1519to2047octets, 542 pkts2048to4095octets, 543 pkts4096to8191octets, 544 pkts8192to10239octets, 545 546 mcx_ppcnt_rfc2819_count 547 }; 548 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 549 550 enum mcx_ppcnt_rfc3635 { 551 dot3stats_alignment_errors, 552 dot3stats_fcs_errors, 553 dot3stats_single_collision_frames, 554 dot3stats_multiple_collision_frames, 555 dot3stats_sqe_test_errors, 556 dot3stats_deferred_transmissions, 557 dot3stats_late_collisions, 558 dot3stats_excessive_collisions, 559 dot3stats_internal_mac_transmit_errors, 560 dot3stats_carrier_sense_errors, 561 dot3stats_frame_too_longs, 562 dot3stats_internal_mac_receive_errors, 563 dot3stats_symbol_errors, 564 dot3control_in_unknown_opcodes, 565 dot3in_pause_frames, 566 dot3out_pause_frames, 567 568 mcx_ppcnt_rfc3635_count 569 }; 570 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 571 572 struct mcx_reg_mtcap { 573 uint8_t _reserved1[3]; 574 uint8_t mtcap_sensor_count; 575 uint8_t _reserved2[4]; 576 577 uint64_t mtcap_sensor_map; 578 }; 579 580 struct mcx_reg_mtmp { 581 uint8_t _reserved1[2]; 582 uint16_t mtmp_sensor_index; 583 584 uint8_t _reserved2[2]; 585 uint16_t mtmp_temperature; 586 587 uint16_t mtmp_mte_mtr; 588 #define MCX_REG_MTMP_MTE (1 << 15) 589 #define MCX_REG_MTMP_MTR (1 << 14) 590 uint16_t mtmp_max_temperature; 591 592 uint16_t mtmp_tee; 593 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 594 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 595 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 596 uint16_t mtmp_temperature_threshold_hi; 597 598 uint8_t _reserved3[2]; 599 uint16_t mtmp_temperature_threshold_lo; 600 601 uint8_t _reserved4[4]; 602 603 uint8_t mtmp_sensor_name[8]; 604 }; 605 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 606 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 607 608 #define MCX_MCIA_EEPROM_BYTES 32 609 struct mcx_reg_mcia { 610 uint8_t rm_l; 611 uint8_t rm_module; 612 uint8_t rm_reserved0; 613 uint8_t rm_status; 614 uint8_t rm_i2c_addr; 615 uint8_t rm_page_num; 616 uint16_t rm_dev_addr; 617 uint16_t rm_reserved1; 618 uint16_t rm_size; 619 uint32_t rm_reserved2; 620 uint8_t rm_data[48]; 621 } __packed __aligned(4); 622 623 struct mcx_cmd_query_issi_in { 624 uint16_t cmd_opcode; 625 uint8_t cmd_reserved0[4]; 626 uint16_t cmd_op_mod; 627 uint8_t cmd_reserved1[8]; 628 } __packed __aligned(4); 629 630 struct mcx_cmd_query_issi_il_out { 631 uint8_t cmd_status; 632 uint8_t cmd_reserved0[3]; 633 uint32_t cmd_syndrome; 634 uint8_t cmd_reserved1[2]; 635 uint16_t cmd_current_issi; 636 uint8_t cmd_reserved2[4]; 637 } __packed __aligned(4); 638 639 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 640 641 struct mcx_cmd_query_issi_mb_out { 642 uint8_t cmd_reserved2[16]; 643 uint8_t cmd_supported_issi[80]; /* very big endian */ 644 } __packed __aligned(4); 645 646 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 647 648 struct mcx_cmd_set_issi_in { 649 uint16_t cmd_opcode; 650 uint8_t cmd_reserved0[4]; 651 uint16_t cmd_op_mod; 652 uint8_t cmd_reserved1[2]; 653 uint16_t cmd_current_issi; 654 uint8_t cmd_reserved2[4]; 655 } __packed __aligned(4); 656 657 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 658 659 struct mcx_cmd_set_issi_out { 660 uint8_t cmd_status; 661 uint8_t cmd_reserved0[3]; 662 uint32_t cmd_syndrome; 663 uint8_t cmd_reserved1[8]; 664 } __packed __aligned(4); 665 666 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 667 668 struct mcx_cmd_query_pages_in { 669 uint16_t cmd_opcode; 670 uint8_t cmd_reserved0[4]; 671 uint16_t cmd_op_mod; 672 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 673 #define MCX_CMD_QUERY_PAGES_INIT 0x02 674 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 675 uint8_t cmd_reserved1[8]; 676 } __packed __aligned(4); 677 678 struct mcx_cmd_query_pages_out { 679 uint8_t cmd_status; 680 uint8_t cmd_reserved0[3]; 681 uint32_t cmd_syndrome; 682 uint8_t cmd_reserved1[2]; 683 uint16_t cmd_func_id; 684 int32_t cmd_num_pages; 685 } __packed __aligned(4); 686 687 struct mcx_cmd_manage_pages_in { 688 uint16_t cmd_opcode; 689 uint8_t cmd_reserved0[4]; 690 uint16_t cmd_op_mod; 691 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 692 0x00 693 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 694 0x01 695 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 696 0x02 697 uint8_t cmd_reserved1[2]; 698 uint16_t cmd_func_id; 699 uint32_t cmd_input_num_entries; 700 } __packed __aligned(4); 701 702 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 703 704 struct mcx_cmd_manage_pages_out { 705 uint8_t cmd_status; 706 uint8_t cmd_reserved0[3]; 707 uint32_t cmd_syndrome; 708 uint32_t cmd_output_num_entries; 709 uint8_t cmd_reserved1[4]; 710 } __packed __aligned(4); 711 712 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 713 714 struct mcx_cmd_query_hca_cap_in { 715 uint16_t cmd_opcode; 716 uint8_t cmd_reserved0[4]; 717 uint16_t cmd_op_mod; 718 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 719 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 720 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 721 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 722 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 723 uint8_t cmd_reserved1[8]; 724 } __packed __aligned(4); 725 726 struct mcx_cmd_query_hca_cap_out { 727 uint8_t cmd_status; 728 uint8_t cmd_reserved0[3]; 729 uint32_t cmd_syndrome; 730 uint8_t cmd_reserved1[8]; 731 } __packed __aligned(4); 732 733 #define MCX_HCA_CAP_LEN 0x1000 734 #define MCX_HCA_CAP_NMAILBOXES \ 735 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 736 737 #if __GNUC_PREREQ__(4, 3) 738 #define __counter__ __COUNTER__ 739 #else 740 #define __counter__ __LINE__ 741 #endif 742 743 #define __token(_tok, _num) _tok##_num 744 #define _token(_tok, _num) __token(_tok, _num) 745 #define __reserved__ _token(__reserved, __counter__) 746 747 struct mcx_cap_device { 748 uint8_t reserved0[16]; 749 750 uint8_t log_max_srq_sz; 751 uint8_t log_max_qp_sz; 752 uint8_t __reserved__[1]; 753 uint8_t log_max_qp; /* 5 bits */ 754 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 755 756 uint8_t __reserved__[1]; 757 uint8_t log_max_srq; /* 5 bits */ 758 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 759 uint8_t __reserved__[2]; 760 761 uint8_t __reserved__[1]; 762 uint8_t log_max_cq_sz; 763 uint8_t __reserved__[1]; 764 uint8_t log_max_cq; /* 5 bits */ 765 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 766 767 uint8_t log_max_eq_sz; 768 uint8_t log_max_mkey; /* 6 bits */ 769 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 770 uint8_t __reserved__[1]; 771 uint8_t log_max_eq; /* 4 bits */ 772 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 773 774 uint8_t max_indirection; 775 uint8_t log_max_mrw_sz; /* 7 bits */ 776 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 777 uint8_t teardown_log_max_msf_list_size; 778 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 779 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 780 0x3f 781 uint8_t log_max_klm_list_size; /* 6 bits */ 782 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 783 0x3f 784 785 uint8_t __reserved__[1]; 786 uint8_t log_max_ra_req_dc; /* 6 bits */ 787 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 788 uint8_t __reserved__[1]; 789 uint8_t log_max_ra_res_dc; /* 6 bits */ 790 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 791 0x3f 792 793 uint8_t __reserved__[1]; 794 uint8_t log_max_ra_req_qp; /* 6 bits */ 795 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 796 0x3f 797 uint8_t __reserved__[1]; 798 uint8_t log_max_ra_res_qp; /* 6 bits */ 799 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 800 0x3f 801 802 uint8_t flags1; 803 #define MCX_CAP_DEVICE_END_PAD 0x80 804 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 805 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 806 0x20 807 #define MCX_CAP_DEVICE_START_PAD 0x10 808 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 809 0x08 810 uint8_t __reserved__[1]; 811 uint16_t gid_table_size; 812 813 uint16_t flags2; 814 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 815 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 816 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 817 0x2000 818 #define MCX_CAP_DEVICE_DEBUG 0x1000 819 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 820 0x8000 821 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 822 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 823 uint16_t pkey_table_size; 824 825 uint8_t flags3; 826 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 827 0x80 828 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 829 0x40 830 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 831 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 832 #define MCX_CAP_DEVICE_ETS 0x04 833 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 834 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 835 0x01 836 uint8_t local_ca_ack_delay; /* 5 bits */ 837 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 838 0x1f 839 uint8_t port_type; 840 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 841 0x80 842 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 843 #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01 844 uint8_t num_ports; 845 846 uint8_t snapshot_log_max_msg; 847 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 848 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 849 uint8_t max_tc; /* 4 bits */ 850 #define MCX_CAP_DEVICE_MAX_TC 0x0f 851 uint8_t flags4; 852 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 853 #define MCX_CAP_DEVICE_DCBX 0x40 854 #define MCX_CAP_DEVICE_ROL_S 0x02 855 #define MCX_CAP_DEVICE_ROL_G 0x01 856 uint8_t wol; 857 #define MCX_CAP_DEVICE_WOL_S 0x40 858 #define MCX_CAP_DEVICE_WOL_G 0x20 859 #define MCX_CAP_DEVICE_WOL_A 0x10 860 #define MCX_CAP_DEVICE_WOL_B 0x08 861 #define MCX_CAP_DEVICE_WOL_M 0x04 862 #define MCX_CAP_DEVICE_WOL_U 0x02 863 #define MCX_CAP_DEVICE_WOL_P 0x01 864 865 uint16_t stat_rate_support; 866 uint8_t __reserved__[1]; 867 uint8_t cqe_version; /* 4 bits */ 868 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 869 870 uint32_t flags5; 871 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 872 0x80000000 873 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 874 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 875 0x10000000 876 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 877 0x08000000 878 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 879 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 880 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 881 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 882 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 883 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 884 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 885 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 886 #define MCX_CAP_DEVICE_SHO 0x00000100 887 #define MCX_CAP_DEVICE_TPH 0x00000080 888 #define MCX_CAP_DEVICE_RF 0x00000040 889 #define MCX_CAP_DEVICE_DCT 0x00000020 890 #define MCX_CAP_DEVICE_QOS 0x00000010 891 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 892 #define MCX_CAP_DEVICE_ROCE 0x00000004 893 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 894 895 uint32_t flags6; 896 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 897 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 898 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 899 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 900 0x10000000 901 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 902 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 903 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 904 #define MCX_CAP_DEVICE_PG 0x01000000 905 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 906 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 907 0x00400000 908 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 909 0x00200000 910 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 911 0x00100000 912 #define MCX_CAP_DEVICE_CD 0x00080000 913 #define MCX_CAP_DEVICE_ATM 0x00040000 914 #define MCX_CAP_DEVICE_APM 0x00020000 915 #define MCX_CAP_DEVICE_IMAICL 0x00010000 916 #define MCX_CAP_DEVICE_QKV 0x00000200 917 #define MCX_CAP_DEVICE_PKV 0x00000100 918 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 919 #define MCX_CAP_DEVICE_XRC 0x00000008 920 #define MCX_CAP_DEVICE_UD 0x00000004 921 #define MCX_CAP_DEVICE_UC 0x00000002 922 #define MCX_CAP_DEVICE_RC 0x00000001 923 924 uint8_t uar_flags; 925 #define MCX_CAP_DEVICE_UAR_4K 0x80 926 uint8_t uar_sz; /* 6 bits */ 927 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 928 uint8_t __reserved__[1]; 929 uint8_t log_pg_sz; 930 931 uint8_t flags7; 932 #define MCX_CAP_DEVICE_BF 0x80 933 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 934 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 935 0x20 936 uint8_t log_bf_reg_size; /* 5 bits */ 937 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 938 uint8_t __reserved__[2]; 939 940 uint16_t num_of_diagnostic_counters; 941 uint16_t max_wqe_sz_sq; 942 943 uint8_t __reserved__[2]; 944 uint16_t max_wqe_sz_rq; 945 946 uint8_t __reserved__[2]; 947 uint16_t max_wqe_sz_sq_dc; 948 949 uint32_t max_qp_mcg; /* 25 bits */ 950 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 951 952 uint8_t __reserved__[3]; 953 uint8_t log_max_mcq; 954 955 uint8_t log_max_transport_domain; /* 5 bits */ 956 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 957 0x1f 958 uint8_t log_max_pd; /* 5 bits */ 959 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 960 uint8_t __reserved__[1]; 961 uint8_t log_max_xrcd; /* 5 bits */ 962 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 963 964 uint8_t __reserved__[2]; 965 uint16_t max_flow_counter; 966 967 uint8_t log_max_rq; /* 5 bits */ 968 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 969 uint8_t log_max_sq; /* 5 bits */ 970 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 971 uint8_t log_max_tir; /* 5 bits */ 972 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 973 uint8_t log_max_tis; /* 5 bits */ 974 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 975 976 uint8_t flags8; 977 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 978 0x80 979 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 980 uint8_t log_max_rqt; /* 5 bits */ 981 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 982 uint8_t log_max_rqt_size; /* 5 bits */ 983 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 984 uint8_t log_max_tis_per_sq; /* 5 bits */ 985 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 986 0x1f 987 988 uint8_t flags9; 989 #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \ 990 0x80 991 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \ 992 0x1f 993 uint8_t log_min_stride_sz_rq; /* 5 bits */ 994 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \ 995 0x1f 996 uint8_t log_max_stride_sz_sq; /* 5 bits */ 997 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \ 998 0x1f 999 uint8_t log_min_stride_sz_sq; /* 5 bits */ 1000 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \ 1001 0x1f 1002 1003 uint8_t log_max_hairpin_queues; 1004 #define MXC_CAP_DEVICE_HAIRPIN 0x80 1005 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \ 1006 0x1f 1007 uint8_t log_min_hairpin_queues; 1008 #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \ 1009 0x1f 1010 uint8_t log_max_hairpin_num_packets; 1011 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \ 1012 0x1f 1013 uint8_t log_max_mq_sz; 1014 #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \ 1015 0x1f 1016 1017 uint8_t log_min_hairpin_wq_data_sz; 1018 #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \ 1019 0x80 1020 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \ 1021 0x40 1022 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \ 1023 0x20 1024 #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \ 1025 0x1f 1026 uint8_t log_max_vlan_list; 1027 #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \ 1028 0x80 1029 #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \ 1030 0x1f 1031 uint8_t log_max_current_mc_list; 1032 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \ 1033 0x1f 1034 uint8_t log_max_current_uc_list; 1035 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \ 1036 0x1f 1037 1038 uint8_t __reserved__[4]; 1039 1040 uint32_t create_qp_start_hint; /* 24 bits */ 1041 1042 uint8_t log_max_uctx; /* 5 bits */ 1043 #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f 1044 uint8_t log_max_umem; /* 5 bits */ 1045 #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f 1046 uint16_t max_num_eqs; 1047 1048 uint8_t log_max_l2_table; /* 5 bits */ 1049 #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f 1050 uint8_t __reserved__[1]; 1051 uint16_t log_uar_page_sz; 1052 1053 uint8_t __reserved__[8]; 1054 1055 uint32_t device_frequency_mhz; 1056 uint32_t device_frequency_khz; 1057 } __packed __aligned(8); 1058 1059 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 1060 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 1061 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 1062 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 1063 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 1064 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 1065 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98); 1066 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c); 1067 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 1068 1069 struct mcx_cmd_set_driver_version_in { 1070 uint16_t cmd_opcode; 1071 uint8_t cmd_reserved0[4]; 1072 uint16_t cmd_op_mod; 1073 uint8_t cmd_reserved1[8]; 1074 } __packed __aligned(4); 1075 1076 struct mcx_cmd_set_driver_version_out { 1077 uint8_t cmd_status; 1078 uint8_t cmd_reserved0[3]; 1079 uint32_t cmd_syndrome; 1080 uint8_t cmd_reserved1[8]; 1081 } __packed __aligned(4); 1082 1083 struct mcx_cmd_set_driver_version { 1084 uint8_t cmd_driver_version[64]; 1085 } __packed __aligned(8); 1086 1087 struct mcx_cmd_modify_nic_vport_context_in { 1088 uint16_t cmd_opcode; 1089 uint8_t cmd_reserved0[4]; 1090 uint16_t cmd_op_mod; 1091 uint8_t cmd_reserved1[4]; 1092 uint32_t cmd_field_select; 1093 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1094 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1095 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1096 } __packed __aligned(4); 1097 1098 struct mcx_cmd_modify_nic_vport_context_out { 1099 uint8_t cmd_status; 1100 uint8_t cmd_reserved0[3]; 1101 uint32_t cmd_syndrome; 1102 uint8_t cmd_reserved1[8]; 1103 } __packed __aligned(4); 1104 1105 struct mcx_cmd_query_nic_vport_context_in { 1106 uint16_t cmd_opcode; 1107 uint8_t cmd_reserved0[4]; 1108 uint16_t cmd_op_mod; 1109 uint8_t cmd_reserved1[4]; 1110 uint8_t cmd_allowed_list_type; 1111 uint8_t cmd_reserved2[3]; 1112 } __packed __aligned(4); 1113 1114 struct mcx_cmd_query_nic_vport_context_out { 1115 uint8_t cmd_status; 1116 uint8_t cmd_reserved0[3]; 1117 uint32_t cmd_syndrome; 1118 uint8_t cmd_reserved1[8]; 1119 } __packed __aligned(4); 1120 1121 struct mcx_nic_vport_ctx { 1122 uint32_t vp_min_wqe_inline_mode; 1123 uint8_t vp_reserved0[32]; 1124 uint32_t vp_mtu; 1125 uint8_t vp_reserved1[200]; 1126 uint16_t vp_flags; 1127 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1128 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1129 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1130 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1131 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1132 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1133 uint16_t vp_allowed_list_size; 1134 uint64_t vp_perm_addr; 1135 uint8_t vp_reserved2[4]; 1136 /* allowed list follows */ 1137 } __packed __aligned(4); 1138 1139 struct mcx_counter { 1140 uint64_t packets; 1141 uint64_t octets; 1142 } __packed __aligned(4); 1143 1144 struct mcx_nic_vport_counters { 1145 struct mcx_counter rx_err; 1146 struct mcx_counter tx_err; 1147 uint8_t reserved0[64]; /* 0x30 */ 1148 struct mcx_counter rx_bcast; 1149 struct mcx_counter tx_bcast; 1150 struct mcx_counter rx_ucast; 1151 struct mcx_counter tx_ucast; 1152 struct mcx_counter rx_mcast; 1153 struct mcx_counter tx_mcast; 1154 uint8_t reserved1[0x210 - 0xd0]; 1155 } __packed __aligned(4); 1156 1157 struct mcx_cmd_query_vport_counters_in { 1158 uint16_t cmd_opcode; 1159 uint8_t cmd_reserved0[4]; 1160 uint16_t cmd_op_mod; 1161 uint8_t cmd_reserved1[8]; 1162 } __packed __aligned(4); 1163 1164 struct mcx_cmd_query_vport_counters_mb_in { 1165 uint8_t cmd_reserved0[8]; 1166 uint8_t cmd_clear; 1167 uint8_t cmd_reserved1[7]; 1168 } __packed __aligned(4); 1169 1170 struct mcx_cmd_query_vport_counters_out { 1171 uint8_t cmd_status; 1172 uint8_t cmd_reserved0[3]; 1173 uint32_t cmd_syndrome; 1174 uint8_t cmd_reserved1[8]; 1175 } __packed __aligned(4); 1176 1177 struct mcx_cmd_query_flow_counter_in { 1178 uint16_t cmd_opcode; 1179 uint8_t cmd_reserved0[4]; 1180 uint16_t cmd_op_mod; 1181 uint8_t cmd_reserved1[8]; 1182 } __packed __aligned(4); 1183 1184 struct mcx_cmd_query_flow_counter_mb_in { 1185 uint8_t cmd_reserved0[8]; 1186 uint8_t cmd_clear; 1187 uint8_t cmd_reserved1[5]; 1188 uint16_t cmd_flow_counter_id; 1189 } __packed __aligned(4); 1190 1191 struct mcx_cmd_query_flow_counter_out { 1192 uint8_t cmd_status; 1193 uint8_t cmd_reserved0[3]; 1194 uint32_t cmd_syndrome; 1195 uint8_t cmd_reserved1[8]; 1196 } __packed __aligned(4); 1197 1198 struct mcx_cmd_alloc_uar_in { 1199 uint16_t cmd_opcode; 1200 uint8_t cmd_reserved0[4]; 1201 uint16_t cmd_op_mod; 1202 uint8_t cmd_reserved1[8]; 1203 } __packed __aligned(4); 1204 1205 struct mcx_cmd_alloc_uar_out { 1206 uint8_t cmd_status; 1207 uint8_t cmd_reserved0[3]; 1208 uint32_t cmd_syndrome; 1209 uint32_t cmd_uar; 1210 uint8_t cmd_reserved1[4]; 1211 } __packed __aligned(4); 1212 1213 struct mcx_cmd_query_special_ctx_in { 1214 uint16_t cmd_opcode; 1215 uint8_t cmd_reserved0[4]; 1216 uint16_t cmd_op_mod; 1217 uint8_t cmd_reserved1[8]; 1218 } __packed __aligned(4); 1219 1220 struct mcx_cmd_query_special_ctx_out { 1221 uint8_t cmd_status; 1222 uint8_t cmd_reserved0[3]; 1223 uint32_t cmd_syndrome; 1224 uint8_t cmd_reserved1[4]; 1225 uint32_t cmd_resd_lkey; 1226 } __packed __aligned(4); 1227 1228 struct mcx_eq_ctx { 1229 uint32_t eq_status; 1230 #define MCX_EQ_CTX_STATE_SHIFT 8 1231 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT) 1232 #define MCX_EQ_CTX_STATE_ARMED 0x9 1233 #define MCX_EQ_CTX_STATE_FIRED 0xa 1234 #define MCX_EQ_CTX_OI_SHIFT 17 1235 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1236 #define MCX_EQ_CTX_EC_SHIFT 18 1237 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1238 #define MCX_EQ_CTX_STATUS_SHIFT 28 1239 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1240 #define MCX_EQ_CTX_STATUS_OK 0x0 1241 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa 1242 uint32_t eq_reserved1; 1243 uint32_t eq_page_offset; 1244 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1245 uint32_t eq_uar_size; 1246 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1247 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1248 uint32_t eq_reserved2; 1249 uint8_t eq_reserved3[3]; 1250 uint8_t eq_intr; 1251 uint32_t eq_log_page_size; 1252 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1253 uint32_t eq_reserved4[3]; 1254 uint32_t eq_consumer_counter; 1255 uint32_t eq_producer_counter; 1256 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1257 uint32_t eq_reserved5[4]; 1258 } __packed __aligned(4); 1259 1260 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1261 1262 struct mcx_cmd_create_eq_in { 1263 uint16_t cmd_opcode; 1264 uint8_t cmd_reserved0[4]; 1265 uint16_t cmd_op_mod; 1266 uint8_t cmd_reserved1[8]; 1267 } __packed __aligned(4); 1268 1269 struct mcx_cmd_create_eq_mb_in { 1270 struct mcx_eq_ctx cmd_eq_ctx; 1271 uint8_t cmd_reserved0[8]; 1272 uint64_t cmd_event_bitmask; 1273 #define MCX_EVENT_TYPE_COMPLETION 0x00 1274 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1275 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1276 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1277 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1278 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1279 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1280 uint8_t cmd_reserved1[176]; 1281 } __packed __aligned(4); 1282 1283 struct mcx_cmd_create_eq_out { 1284 uint8_t cmd_status; 1285 uint8_t cmd_reserved0[3]; 1286 uint32_t cmd_syndrome; 1287 uint32_t cmd_eqn; 1288 uint8_t cmd_reserved1[4]; 1289 } __packed __aligned(4); 1290 1291 struct mcx_cmd_query_eq_in { 1292 uint16_t cmd_opcode; 1293 uint8_t cmd_reserved0[4]; 1294 uint16_t cmd_op_mod; 1295 uint32_t cmd_eqn; 1296 uint8_t cmd_reserved1[4]; 1297 } __packed __aligned(4); 1298 1299 struct mcx_cmd_query_eq_out { 1300 uint8_t cmd_status; 1301 uint8_t cmd_reserved0[3]; 1302 uint32_t cmd_syndrome; 1303 uint8_t cmd_reserved1[8]; 1304 } __packed __aligned(4); 1305 1306 struct mcx_eq_entry { 1307 uint8_t eq_reserved1; 1308 uint8_t eq_event_type; 1309 uint8_t eq_reserved2; 1310 uint8_t eq_event_sub_type; 1311 1312 uint8_t eq_reserved3[28]; 1313 uint32_t eq_event_data[7]; 1314 uint8_t eq_reserved4[2]; 1315 uint8_t eq_signature; 1316 uint8_t eq_owner; 1317 #define MCX_EQ_ENTRY_OWNER_INIT 1 1318 } __packed __aligned(4); 1319 1320 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1321 1322 struct mcx_cmd_alloc_pd_in { 1323 uint16_t cmd_opcode; 1324 uint8_t cmd_reserved0[4]; 1325 uint16_t cmd_op_mod; 1326 uint8_t cmd_reserved1[8]; 1327 } __packed __aligned(4); 1328 1329 struct mcx_cmd_alloc_pd_out { 1330 uint8_t cmd_status; 1331 uint8_t cmd_reserved0[3]; 1332 uint32_t cmd_syndrome; 1333 uint32_t cmd_pd; 1334 uint8_t cmd_reserved1[4]; 1335 } __packed __aligned(4); 1336 1337 struct mcx_cmd_alloc_td_in { 1338 uint16_t cmd_opcode; 1339 uint8_t cmd_reserved0[4]; 1340 uint16_t cmd_op_mod; 1341 uint8_t cmd_reserved1[8]; 1342 } __packed __aligned(4); 1343 1344 struct mcx_cmd_alloc_td_out { 1345 uint8_t cmd_status; 1346 uint8_t cmd_reserved0[3]; 1347 uint32_t cmd_syndrome; 1348 uint32_t cmd_tdomain; 1349 uint8_t cmd_reserved1[4]; 1350 } __packed __aligned(4); 1351 1352 struct mcx_cmd_create_tir_in { 1353 uint16_t cmd_opcode; 1354 uint8_t cmd_reserved0[4]; 1355 uint16_t cmd_op_mod; 1356 uint8_t cmd_reserved1[8]; 1357 } __packed __aligned(4); 1358 1359 struct mcx_cmd_create_tir_mb_in { 1360 uint8_t cmd_reserved0[20]; 1361 uint32_t cmd_disp_type; 1362 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1363 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1364 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1365 uint8_t cmd_reserved1[8]; 1366 uint32_t cmd_lro; 1367 uint8_t cmd_reserved2[8]; 1368 uint32_t cmd_inline_rqn; 1369 uint32_t cmd_indir_table; 1370 uint32_t cmd_tdomain; 1371 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1372 #define MCX_TIR_CTX_HASH_SHIFT 28 1373 uint8_t cmd_rx_hash_key[40]; 1374 uint32_t cmd_rx_hash_sel_outer; 1375 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1376 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1377 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1378 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1379 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1380 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1381 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1382 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1383 uint32_t cmd_rx_hash_sel_inner; 1384 uint8_t cmd_reserved3[152]; 1385 } __packed __aligned(4); 1386 1387 struct mcx_cmd_create_tir_out { 1388 uint8_t cmd_status; 1389 uint8_t cmd_reserved0[3]; 1390 uint32_t cmd_syndrome; 1391 uint32_t cmd_tirn; 1392 uint8_t cmd_reserved1[4]; 1393 } __packed __aligned(4); 1394 1395 struct mcx_cmd_destroy_tir_in { 1396 uint16_t cmd_opcode; 1397 uint8_t cmd_reserved0[4]; 1398 uint16_t cmd_op_mod; 1399 uint32_t cmd_tirn; 1400 uint8_t cmd_reserved1[4]; 1401 } __packed __aligned(4); 1402 1403 struct mcx_cmd_destroy_tir_out { 1404 uint8_t cmd_status; 1405 uint8_t cmd_reserved0[3]; 1406 uint32_t cmd_syndrome; 1407 uint8_t cmd_reserved1[8]; 1408 } __packed __aligned(4); 1409 1410 struct mcx_cmd_create_tis_in { 1411 uint16_t cmd_opcode; 1412 uint8_t cmd_reserved0[4]; 1413 uint16_t cmd_op_mod; 1414 uint8_t cmd_reserved1[8]; 1415 } __packed __aligned(4); 1416 1417 struct mcx_cmd_create_tis_mb_in { 1418 uint8_t cmd_reserved[16]; 1419 uint32_t cmd_prio; 1420 uint8_t cmd_reserved1[32]; 1421 uint32_t cmd_tdomain; 1422 uint8_t cmd_reserved2[120]; 1423 } __packed __aligned(4); 1424 1425 struct mcx_cmd_create_tis_out { 1426 uint8_t cmd_status; 1427 uint8_t cmd_reserved0[3]; 1428 uint32_t cmd_syndrome; 1429 uint32_t cmd_tisn; 1430 uint8_t cmd_reserved1[4]; 1431 } __packed __aligned(4); 1432 1433 struct mcx_cmd_destroy_tis_in { 1434 uint16_t cmd_opcode; 1435 uint8_t cmd_reserved0[4]; 1436 uint16_t cmd_op_mod; 1437 uint32_t cmd_tisn; 1438 uint8_t cmd_reserved1[4]; 1439 } __packed __aligned(4); 1440 1441 struct mcx_cmd_destroy_tis_out { 1442 uint8_t cmd_status; 1443 uint8_t cmd_reserved0[3]; 1444 uint32_t cmd_syndrome; 1445 uint8_t cmd_reserved1[8]; 1446 } __packed __aligned(4); 1447 1448 struct mcx_cmd_create_rqt_in { 1449 uint16_t cmd_opcode; 1450 uint8_t cmd_reserved0[4]; 1451 uint16_t cmd_op_mod; 1452 uint8_t cmd_reserved1[8]; 1453 } __packed __aligned(4); 1454 1455 struct mcx_rqt_ctx { 1456 uint8_t cmd_reserved0[20]; 1457 uint16_t cmd_reserved1; 1458 uint16_t cmd_rqt_max_size; 1459 uint16_t cmd_reserved2; 1460 uint16_t cmd_rqt_actual_size; 1461 uint8_t cmd_reserved3[212]; 1462 } __packed __aligned(4); 1463 1464 struct mcx_cmd_create_rqt_mb_in { 1465 uint8_t cmd_reserved0[16]; 1466 struct mcx_rqt_ctx cmd_rqt; 1467 } __packed __aligned(4); 1468 1469 struct mcx_cmd_create_rqt_out { 1470 uint8_t cmd_status; 1471 uint8_t cmd_reserved0[3]; 1472 uint32_t cmd_syndrome; 1473 uint32_t cmd_rqtn; 1474 uint8_t cmd_reserved1[4]; 1475 } __packed __aligned(4); 1476 1477 struct mcx_cmd_destroy_rqt_in { 1478 uint16_t cmd_opcode; 1479 uint8_t cmd_reserved0[4]; 1480 uint16_t cmd_op_mod; 1481 uint32_t cmd_rqtn; 1482 uint8_t cmd_reserved1[4]; 1483 } __packed __aligned(4); 1484 1485 struct mcx_cmd_destroy_rqt_out { 1486 uint8_t cmd_status; 1487 uint8_t cmd_reserved0[3]; 1488 uint32_t cmd_syndrome; 1489 uint8_t cmd_reserved1[8]; 1490 } __packed __aligned(4); 1491 1492 struct mcx_cq_ctx { 1493 uint32_t cq_status; 1494 #define MCX_CQ_CTX_STATUS_SHIFT 28 1495 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT) 1496 #define MCX_CQ_CTX_STATUS_OK 0x0 1497 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9 1498 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa 1499 #define MCX_CQ_CTX_STATE_SHIFT 8 1500 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT) 1501 #define MCX_CQ_CTX_STATE_SOLICITED 0x6 1502 #define MCX_CQ_CTX_STATE_ARMED 0x9 1503 #define MCX_CQ_CTX_STATE_FIRED 0xa 1504 uint32_t cq_reserved1; 1505 uint32_t cq_page_offset; 1506 uint32_t cq_uar_size; 1507 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1508 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1509 uint32_t cq_period_max_count; 1510 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1511 uint32_t cq_eqn; 1512 uint32_t cq_log_page_size; 1513 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1514 uint32_t cq_reserved2; 1515 uint32_t cq_last_notified; 1516 uint32_t cq_last_solicit; 1517 uint32_t cq_consumer_counter; 1518 uint32_t cq_producer_counter; 1519 uint8_t cq_reserved3[8]; 1520 uint64_t cq_doorbell; 1521 } __packed __aligned(4); 1522 1523 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1524 1525 struct mcx_cmd_create_cq_in { 1526 uint16_t cmd_opcode; 1527 uint8_t cmd_reserved0[4]; 1528 uint16_t cmd_op_mod; 1529 uint8_t cmd_reserved1[8]; 1530 } __packed __aligned(4); 1531 1532 struct mcx_cmd_create_cq_mb_in { 1533 struct mcx_cq_ctx cmd_cq_ctx; 1534 uint8_t cmd_reserved1[192]; 1535 } __packed __aligned(4); 1536 1537 struct mcx_cmd_create_cq_out { 1538 uint8_t cmd_status; 1539 uint8_t cmd_reserved0[3]; 1540 uint32_t cmd_syndrome; 1541 uint32_t cmd_cqn; 1542 uint8_t cmd_reserved1[4]; 1543 } __packed __aligned(4); 1544 1545 struct mcx_cmd_destroy_cq_in { 1546 uint16_t cmd_opcode; 1547 uint8_t cmd_reserved0[4]; 1548 uint16_t cmd_op_mod; 1549 uint32_t cmd_cqn; 1550 uint8_t cmd_reserved1[4]; 1551 } __packed __aligned(4); 1552 1553 struct mcx_cmd_destroy_cq_out { 1554 uint8_t cmd_status; 1555 uint8_t cmd_reserved0[3]; 1556 uint32_t cmd_syndrome; 1557 uint8_t cmd_reserved1[8]; 1558 } __packed __aligned(4); 1559 1560 struct mcx_cmd_query_cq_in { 1561 uint16_t cmd_opcode; 1562 uint8_t cmd_reserved0[4]; 1563 uint16_t cmd_op_mod; 1564 uint32_t cmd_cqn; 1565 uint8_t cmd_reserved1[4]; 1566 } __packed __aligned(4); 1567 1568 struct mcx_cmd_query_cq_out { 1569 uint8_t cmd_status; 1570 uint8_t cmd_reserved0[3]; 1571 uint32_t cmd_syndrome; 1572 uint8_t cmd_reserved1[8]; 1573 } __packed __aligned(4); 1574 1575 struct mcx_cq_entry { 1576 uint32_t __reserved__; 1577 uint32_t cq_lro; 1578 uint32_t cq_lro_ack_seq_num; 1579 uint32_t cq_rx_hash; 1580 uint8_t cq_rx_hash_type; 1581 uint8_t cq_ml_path; 1582 uint16_t __reserved__; 1583 uint32_t cq_checksum; 1584 uint32_t __reserved__; 1585 uint32_t cq_flags; 1586 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1587 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1588 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1589 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1590 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1591 1592 uint32_t cq_lro_srqn; 1593 uint32_t __reserved__[2]; 1594 uint32_t cq_byte_cnt; 1595 uint64_t cq_timestamp; 1596 uint8_t cq_rx_drops; 1597 uint8_t cq_flow_tag[3]; 1598 uint16_t cq_wqe_count; 1599 uint8_t cq_signature; 1600 uint8_t cq_opcode_owner; 1601 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1602 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1603 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1604 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1605 1606 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1607 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1608 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1609 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1610 1611 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1612 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1613 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1614 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1615 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1616 1617 } __packed __aligned(4); 1618 1619 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1620 1621 struct mcx_cq_doorbell { 1622 uint32_t db_update_ci; 1623 uint32_t db_arm_ci; 1624 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1625 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1626 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1627 } __packed __aligned(8); 1628 1629 struct mcx_wq_ctx { 1630 uint8_t wq_type; 1631 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1632 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1633 uint8_t wq_reserved0[5]; 1634 uint16_t wq_lwm; 1635 uint32_t wq_pd; 1636 uint32_t wq_uar_page; 1637 uint64_t wq_doorbell; 1638 uint32_t wq_hw_counter; 1639 uint32_t wq_sw_counter; 1640 uint16_t wq_log_stride; 1641 uint8_t wq_log_page_sz; 1642 uint8_t wq_log_size; 1643 uint8_t wq_reserved1[156]; 1644 } __packed __aligned(4); 1645 1646 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1647 1648 struct mcx_sq_ctx { 1649 uint32_t sq_flags; 1650 #define MCX_SQ_CTX_RLKEY (1 << 31) 1651 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1652 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1653 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1654 #define MCX_SQ_CTX_STATE_SHIFT 20 1655 #define MCX_SQ_CTX_STATE_MASK (0xf << 20) 1656 #define MCX_SQ_CTX_STATE_RST 0 1657 #define MCX_SQ_CTX_STATE_RDY 1 1658 #define MCX_SQ_CTX_STATE_ERR 3 1659 uint32_t sq_user_index; 1660 uint32_t sq_cqn; 1661 uint32_t sq_reserved1[5]; 1662 uint32_t sq_tis_lst_sz; 1663 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1664 uint32_t sq_reserved2[2]; 1665 uint32_t sq_tis_num; 1666 struct mcx_wq_ctx sq_wq; 1667 } __packed __aligned(4); 1668 1669 struct mcx_sq_entry_seg { 1670 uint32_t sqs_byte_count; 1671 uint32_t sqs_lkey; 1672 uint64_t sqs_addr; 1673 } __packed __aligned(4); 1674 1675 struct mcx_sq_entry { 1676 /* control segment */ 1677 uint32_t sqe_opcode_index; 1678 #define MCX_SQE_WQE_INDEX_SHIFT 8 1679 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1680 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1681 uint32_t sqe_ds_sq_num; 1682 #define MCX_SQE_SQ_NUM_SHIFT 8 1683 uint32_t sqe_signature; 1684 #define MCX_SQE_SIGNATURE_SHIFT 24 1685 #define MCX_SQE_SOLICITED_EVENT 0x02 1686 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1687 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1688 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1689 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1690 #define MCX_SQE_FM_NO_FENCE 0x00 1691 #define MCX_SQE_FM_SMALL_FENCE 0x40 1692 uint32_t sqe_mkey; 1693 1694 /* ethernet segment */ 1695 uint32_t sqe_reserved1; 1696 uint32_t sqe_mss_csum; 1697 #define MCX_SQE_L4_CSUM (1 << 31) 1698 #define MCX_SQE_L3_CSUM (1 << 30) 1699 uint32_t sqe_reserved2; 1700 uint16_t sqe_inline_header_size; 1701 uint16_t sqe_inline_headers[9]; 1702 1703 /* data segment */ 1704 struct mcx_sq_entry_seg sqe_segs[1]; 1705 } __packed __aligned(64); 1706 1707 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1708 1709 struct mcx_cmd_create_sq_in { 1710 uint16_t cmd_opcode; 1711 uint8_t cmd_reserved0[4]; 1712 uint16_t cmd_op_mod; 1713 uint8_t cmd_reserved1[8]; 1714 } __packed __aligned(4); 1715 1716 struct mcx_cmd_create_sq_out { 1717 uint8_t cmd_status; 1718 uint8_t cmd_reserved0[3]; 1719 uint32_t cmd_syndrome; 1720 uint32_t cmd_sqn; 1721 uint8_t cmd_reserved1[4]; 1722 } __packed __aligned(4); 1723 1724 struct mcx_cmd_modify_sq_in { 1725 uint16_t cmd_opcode; 1726 uint8_t cmd_reserved0[4]; 1727 uint16_t cmd_op_mod; 1728 uint32_t cmd_sq_state; 1729 uint8_t cmd_reserved1[4]; 1730 } __packed __aligned(4); 1731 1732 struct mcx_cmd_modify_sq_mb_in { 1733 uint32_t cmd_modify_hi; 1734 uint32_t cmd_modify_lo; 1735 uint8_t cmd_reserved0[8]; 1736 struct mcx_sq_ctx cmd_sq_ctx; 1737 } __packed __aligned(4); 1738 1739 struct mcx_cmd_modify_sq_out { 1740 uint8_t cmd_status; 1741 uint8_t cmd_reserved0[3]; 1742 uint32_t cmd_syndrome; 1743 uint8_t cmd_reserved1[8]; 1744 } __packed __aligned(4); 1745 1746 struct mcx_cmd_destroy_sq_in { 1747 uint16_t cmd_opcode; 1748 uint8_t cmd_reserved0[4]; 1749 uint16_t cmd_op_mod; 1750 uint32_t cmd_sqn; 1751 uint8_t cmd_reserved1[4]; 1752 } __packed __aligned(4); 1753 1754 struct mcx_cmd_destroy_sq_out { 1755 uint8_t cmd_status; 1756 uint8_t cmd_reserved0[3]; 1757 uint32_t cmd_syndrome; 1758 uint8_t cmd_reserved1[8]; 1759 } __packed __aligned(4); 1760 1761 1762 struct mcx_rq_ctx { 1763 uint32_t rq_flags; 1764 #define MCX_RQ_CTX_RLKEY (1 << 31) 1765 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1766 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1767 #define MCX_RQ_CTX_STATE_SHIFT 20 1768 #define MCX_RQ_CTX_STATE_MASK (0xf << 20) 1769 #define MCX_RQ_CTX_STATE_RST 0 1770 #define MCX_RQ_CTX_STATE_RDY 1 1771 #define MCX_RQ_CTX_STATE_ERR 3 1772 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1773 uint32_t rq_user_index; 1774 uint32_t rq_cqn; 1775 uint32_t rq_reserved1; 1776 uint32_t rq_rmpn; 1777 uint32_t rq_reserved2[7]; 1778 struct mcx_wq_ctx rq_wq; 1779 } __packed __aligned(4); 1780 1781 struct mcx_rq_entry { 1782 uint32_t rqe_byte_count; 1783 uint32_t rqe_lkey; 1784 uint64_t rqe_addr; 1785 } __packed __aligned(16); 1786 1787 struct mcx_cmd_create_rq_in { 1788 uint16_t cmd_opcode; 1789 uint8_t cmd_reserved0[4]; 1790 uint16_t cmd_op_mod; 1791 uint8_t cmd_reserved1[8]; 1792 } __packed __aligned(4); 1793 1794 struct mcx_cmd_create_rq_out { 1795 uint8_t cmd_status; 1796 uint8_t cmd_reserved0[3]; 1797 uint32_t cmd_syndrome; 1798 uint32_t cmd_rqn; 1799 uint8_t cmd_reserved1[4]; 1800 } __packed __aligned(4); 1801 1802 struct mcx_cmd_modify_rq_in { 1803 uint16_t cmd_opcode; 1804 uint8_t cmd_reserved0[4]; 1805 uint16_t cmd_op_mod; 1806 uint32_t cmd_rq_state; 1807 uint8_t cmd_reserved1[4]; 1808 } __packed __aligned(4); 1809 1810 struct mcx_cmd_modify_rq_mb_in { 1811 uint32_t cmd_modify_hi; 1812 uint32_t cmd_modify_lo; 1813 uint8_t cmd_reserved0[8]; 1814 struct mcx_rq_ctx cmd_rq_ctx; 1815 } __packed __aligned(4); 1816 1817 struct mcx_cmd_modify_rq_out { 1818 uint8_t cmd_status; 1819 uint8_t cmd_reserved0[3]; 1820 uint32_t cmd_syndrome; 1821 uint8_t cmd_reserved1[8]; 1822 } __packed __aligned(4); 1823 1824 struct mcx_cmd_destroy_rq_in { 1825 uint16_t cmd_opcode; 1826 uint8_t cmd_reserved0[4]; 1827 uint16_t cmd_op_mod; 1828 uint32_t cmd_rqn; 1829 uint8_t cmd_reserved1[4]; 1830 } __packed __aligned(4); 1831 1832 struct mcx_cmd_destroy_rq_out { 1833 uint8_t cmd_status; 1834 uint8_t cmd_reserved0[3]; 1835 uint32_t cmd_syndrome; 1836 uint8_t cmd_reserved1[8]; 1837 } __packed __aligned(4); 1838 1839 struct mcx_cmd_create_flow_table_in { 1840 uint16_t cmd_opcode; 1841 uint8_t cmd_reserved0[4]; 1842 uint16_t cmd_op_mod; 1843 uint8_t cmd_reserved1[8]; 1844 } __packed __aligned(4); 1845 1846 struct mcx_flow_table_ctx { 1847 uint8_t ft_miss_action; 1848 uint8_t ft_level; 1849 uint8_t ft_reserved0; 1850 uint8_t ft_log_size; 1851 uint32_t ft_table_miss_id; 1852 uint8_t ft_reserved1[28]; 1853 } __packed __aligned(4); 1854 1855 struct mcx_cmd_create_flow_table_mb_in { 1856 uint8_t cmd_table_type; 1857 uint8_t cmd_reserved0[7]; 1858 struct mcx_flow_table_ctx cmd_ctx; 1859 } __packed __aligned(4); 1860 1861 struct mcx_cmd_create_flow_table_out { 1862 uint8_t cmd_status; 1863 uint8_t cmd_reserved0[3]; 1864 uint32_t cmd_syndrome; 1865 uint32_t cmd_table_id; 1866 uint8_t cmd_reserved1[4]; 1867 } __packed __aligned(4); 1868 1869 struct mcx_cmd_destroy_flow_table_in { 1870 uint16_t cmd_opcode; 1871 uint8_t cmd_reserved0[4]; 1872 uint16_t cmd_op_mod; 1873 uint8_t cmd_reserved1[8]; 1874 } __packed __aligned(4); 1875 1876 struct mcx_cmd_destroy_flow_table_mb_in { 1877 uint8_t cmd_table_type; 1878 uint8_t cmd_reserved0[3]; 1879 uint32_t cmd_table_id; 1880 uint8_t cmd_reserved1[40]; 1881 } __packed __aligned(4); 1882 1883 struct mcx_cmd_destroy_flow_table_out { 1884 uint8_t cmd_status; 1885 uint8_t cmd_reserved0[3]; 1886 uint32_t cmd_syndrome; 1887 uint8_t cmd_reserved1[8]; 1888 } __packed __aligned(4); 1889 1890 struct mcx_cmd_set_flow_table_root_in { 1891 uint16_t cmd_opcode; 1892 uint8_t cmd_reserved0[4]; 1893 uint16_t cmd_op_mod; 1894 uint8_t cmd_reserved1[8]; 1895 } __packed __aligned(4); 1896 1897 struct mcx_cmd_set_flow_table_root_mb_in { 1898 uint8_t cmd_table_type; 1899 uint8_t cmd_reserved0[3]; 1900 uint32_t cmd_table_id; 1901 uint8_t cmd_reserved1[56]; 1902 } __packed __aligned(4); 1903 1904 struct mcx_cmd_set_flow_table_root_out { 1905 uint8_t cmd_status; 1906 uint8_t cmd_reserved0[3]; 1907 uint32_t cmd_syndrome; 1908 uint8_t cmd_reserved1[8]; 1909 } __packed __aligned(4); 1910 1911 struct mcx_flow_match { 1912 /* outer headers */ 1913 uint8_t mc_src_mac[6]; 1914 uint16_t mc_ethertype; 1915 uint8_t mc_dest_mac[6]; 1916 uint16_t mc_first_vlan; 1917 uint8_t mc_ip_proto; 1918 uint8_t mc_ip_dscp_ecn; 1919 uint8_t mc_vlan_flags; 1920 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1921 uint8_t mc_tcp_flags; 1922 uint16_t mc_tcp_sport; 1923 uint16_t mc_tcp_dport; 1924 uint32_t mc_reserved0; 1925 uint16_t mc_udp_sport; 1926 uint16_t mc_udp_dport; 1927 uint8_t mc_src_ip[16]; 1928 uint8_t mc_dest_ip[16]; 1929 1930 /* misc parameters */ 1931 uint8_t mc_reserved1[8]; 1932 uint16_t mc_second_vlan; 1933 uint8_t mc_reserved2[2]; 1934 uint8_t mc_second_vlan_flags; 1935 uint8_t mc_reserved3[15]; 1936 uint32_t mc_outer_ipv6_flow_label; 1937 uint8_t mc_reserved4[32]; 1938 1939 uint8_t mc_reserved[384]; 1940 } __packed __aligned(4); 1941 1942 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1943 1944 struct mcx_cmd_create_flow_group_in { 1945 uint16_t cmd_opcode; 1946 uint8_t cmd_reserved0[4]; 1947 uint16_t cmd_op_mod; 1948 uint8_t cmd_reserved1[8]; 1949 } __packed __aligned(4); 1950 1951 struct mcx_cmd_create_flow_group_mb_in { 1952 uint8_t cmd_table_type; 1953 uint8_t cmd_reserved0[3]; 1954 uint32_t cmd_table_id; 1955 uint8_t cmd_reserved1[4]; 1956 uint32_t cmd_start_flow_index; 1957 uint8_t cmd_reserved2[4]; 1958 uint32_t cmd_end_flow_index; 1959 uint8_t cmd_reserved3[23]; 1960 uint8_t cmd_match_criteria_enable; 1961 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1962 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1963 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1964 struct mcx_flow_match cmd_match_criteria; 1965 uint8_t cmd_reserved4[448]; 1966 } __packed __aligned(4); 1967 1968 struct mcx_cmd_create_flow_group_out { 1969 uint8_t cmd_status; 1970 uint8_t cmd_reserved0[3]; 1971 uint32_t cmd_syndrome; 1972 uint32_t cmd_group_id; 1973 uint8_t cmd_reserved1[4]; 1974 } __packed __aligned(4); 1975 1976 struct mcx_flow_ctx { 1977 uint8_t fc_reserved0[4]; 1978 uint32_t fc_group_id; 1979 uint32_t fc_flow_tag; 1980 uint32_t fc_action; 1981 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 1982 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 1983 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 1984 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 1985 uint32_t fc_dest_list_size; 1986 uint32_t fc_counter_list_size; 1987 uint8_t fc_reserved1[40]; 1988 struct mcx_flow_match fc_match_value; 1989 uint8_t fc_reserved2[192]; 1990 } __packed __aligned(4); 1991 1992 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 1993 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 1994 1995 struct mcx_cmd_destroy_flow_group_in { 1996 uint16_t cmd_opcode; 1997 uint8_t cmd_reserved0[4]; 1998 uint16_t cmd_op_mod; 1999 uint8_t cmd_reserved1[8]; 2000 } __packed __aligned(4); 2001 2002 struct mcx_cmd_destroy_flow_group_mb_in { 2003 uint8_t cmd_table_type; 2004 uint8_t cmd_reserved0[3]; 2005 uint32_t cmd_table_id; 2006 uint32_t cmd_group_id; 2007 uint8_t cmd_reserved1[36]; 2008 } __packed __aligned(4); 2009 2010 struct mcx_cmd_destroy_flow_group_out { 2011 uint8_t cmd_status; 2012 uint8_t cmd_reserved0[3]; 2013 uint32_t cmd_syndrome; 2014 uint8_t cmd_reserved1[8]; 2015 } __packed __aligned(4); 2016 2017 struct mcx_cmd_set_flow_table_entry_in { 2018 uint16_t cmd_opcode; 2019 uint8_t cmd_reserved0[4]; 2020 uint16_t cmd_op_mod; 2021 uint8_t cmd_reserved1[8]; 2022 } __packed __aligned(4); 2023 2024 struct mcx_cmd_set_flow_table_entry_mb_in { 2025 uint8_t cmd_table_type; 2026 uint8_t cmd_reserved0[3]; 2027 uint32_t cmd_table_id; 2028 uint32_t cmd_modify_enable_mask; 2029 uint8_t cmd_reserved1[4]; 2030 uint32_t cmd_flow_index; 2031 uint8_t cmd_reserved2[28]; 2032 struct mcx_flow_ctx cmd_flow_ctx; 2033 } __packed __aligned(4); 2034 2035 struct mcx_cmd_set_flow_table_entry_out { 2036 uint8_t cmd_status; 2037 uint8_t cmd_reserved0[3]; 2038 uint32_t cmd_syndrome; 2039 uint8_t cmd_reserved1[8]; 2040 } __packed __aligned(4); 2041 2042 struct mcx_cmd_query_flow_table_entry_in { 2043 uint16_t cmd_opcode; 2044 uint8_t cmd_reserved0[4]; 2045 uint16_t cmd_op_mod; 2046 uint8_t cmd_reserved1[8]; 2047 } __packed __aligned(4); 2048 2049 struct mcx_cmd_query_flow_table_entry_mb_in { 2050 uint8_t cmd_table_type; 2051 uint8_t cmd_reserved0[3]; 2052 uint32_t cmd_table_id; 2053 uint8_t cmd_reserved1[8]; 2054 uint32_t cmd_flow_index; 2055 uint8_t cmd_reserved2[28]; 2056 } __packed __aligned(4); 2057 2058 struct mcx_cmd_query_flow_table_entry_out { 2059 uint8_t cmd_status; 2060 uint8_t cmd_reserved0[3]; 2061 uint32_t cmd_syndrome; 2062 uint8_t cmd_reserved1[8]; 2063 } __packed __aligned(4); 2064 2065 struct mcx_cmd_query_flow_table_entry_mb_out { 2066 uint8_t cmd_reserved0[48]; 2067 struct mcx_flow_ctx cmd_flow_ctx; 2068 } __packed __aligned(4); 2069 2070 struct mcx_cmd_delete_flow_table_entry_in { 2071 uint16_t cmd_opcode; 2072 uint8_t cmd_reserved0[4]; 2073 uint16_t cmd_op_mod; 2074 uint8_t cmd_reserved1[8]; 2075 } __packed __aligned(4); 2076 2077 struct mcx_cmd_delete_flow_table_entry_mb_in { 2078 uint8_t cmd_table_type; 2079 uint8_t cmd_reserved0[3]; 2080 uint32_t cmd_table_id; 2081 uint8_t cmd_reserved1[8]; 2082 uint32_t cmd_flow_index; 2083 uint8_t cmd_reserved2[28]; 2084 } __packed __aligned(4); 2085 2086 struct mcx_cmd_delete_flow_table_entry_out { 2087 uint8_t cmd_status; 2088 uint8_t cmd_reserved0[3]; 2089 uint32_t cmd_syndrome; 2090 uint8_t cmd_reserved1[8]; 2091 } __packed __aligned(4); 2092 2093 struct mcx_cmd_query_flow_group_in { 2094 uint16_t cmd_opcode; 2095 uint8_t cmd_reserved0[4]; 2096 uint16_t cmd_op_mod; 2097 uint8_t cmd_reserved1[8]; 2098 } __packed __aligned(4); 2099 2100 struct mcx_cmd_query_flow_group_mb_in { 2101 uint8_t cmd_table_type; 2102 uint8_t cmd_reserved0[3]; 2103 uint32_t cmd_table_id; 2104 uint32_t cmd_group_id; 2105 uint8_t cmd_reserved1[36]; 2106 } __packed __aligned(4); 2107 2108 struct mcx_cmd_query_flow_group_out { 2109 uint8_t cmd_status; 2110 uint8_t cmd_reserved0[3]; 2111 uint32_t cmd_syndrome; 2112 uint8_t cmd_reserved1[8]; 2113 } __packed __aligned(4); 2114 2115 struct mcx_cmd_query_flow_group_mb_out { 2116 uint8_t cmd_reserved0[12]; 2117 uint32_t cmd_start_flow_index; 2118 uint8_t cmd_reserved1[4]; 2119 uint32_t cmd_end_flow_index; 2120 uint8_t cmd_reserved2[20]; 2121 uint32_t cmd_match_criteria_enable; 2122 uint8_t cmd_match_criteria[512]; 2123 uint8_t cmd_reserved4[448]; 2124 } __packed __aligned(4); 2125 2126 struct mcx_cmd_query_flow_table_in { 2127 uint16_t cmd_opcode; 2128 uint8_t cmd_reserved0[4]; 2129 uint16_t cmd_op_mod; 2130 uint8_t cmd_reserved1[8]; 2131 } __packed __aligned(4); 2132 2133 struct mcx_cmd_query_flow_table_mb_in { 2134 uint8_t cmd_table_type; 2135 uint8_t cmd_reserved0[3]; 2136 uint32_t cmd_table_id; 2137 uint8_t cmd_reserved1[40]; 2138 } __packed __aligned(4); 2139 2140 struct mcx_cmd_query_flow_table_out { 2141 uint8_t cmd_status; 2142 uint8_t cmd_reserved0[3]; 2143 uint32_t cmd_syndrome; 2144 uint8_t cmd_reserved1[8]; 2145 } __packed __aligned(4); 2146 2147 struct mcx_cmd_query_flow_table_mb_out { 2148 uint8_t cmd_reserved0[4]; 2149 struct mcx_flow_table_ctx cmd_ctx; 2150 } __packed __aligned(4); 2151 2152 struct mcx_cmd_alloc_flow_counter_in { 2153 uint16_t cmd_opcode; 2154 uint8_t cmd_reserved0[4]; 2155 uint16_t cmd_op_mod; 2156 uint8_t cmd_reserved1[8]; 2157 } __packed __aligned(4); 2158 2159 struct mcx_cmd_query_rq_in { 2160 uint16_t cmd_opcode; 2161 uint8_t cmd_reserved0[4]; 2162 uint16_t cmd_op_mod; 2163 uint32_t cmd_rqn; 2164 uint8_t cmd_reserved1[4]; 2165 } __packed __aligned(4); 2166 2167 struct mcx_cmd_query_rq_out { 2168 uint8_t cmd_status; 2169 uint8_t cmd_reserved0[3]; 2170 uint32_t cmd_syndrome; 2171 uint8_t cmd_reserved1[8]; 2172 } __packed __aligned(4); 2173 2174 struct mcx_cmd_query_rq_mb_out { 2175 uint8_t cmd_reserved0[16]; 2176 struct mcx_rq_ctx cmd_ctx; 2177 }; 2178 2179 struct mcx_cmd_query_sq_in { 2180 uint16_t cmd_opcode; 2181 uint8_t cmd_reserved0[4]; 2182 uint16_t cmd_op_mod; 2183 uint32_t cmd_sqn; 2184 uint8_t cmd_reserved1[4]; 2185 } __packed __aligned(4); 2186 2187 struct mcx_cmd_query_sq_out { 2188 uint8_t cmd_status; 2189 uint8_t cmd_reserved0[3]; 2190 uint32_t cmd_syndrome; 2191 uint8_t cmd_reserved1[8]; 2192 } __packed __aligned(4); 2193 2194 struct mcx_cmd_query_sq_mb_out { 2195 uint8_t cmd_reserved0[16]; 2196 struct mcx_sq_ctx cmd_ctx; 2197 }; 2198 2199 struct mcx_cmd_alloc_flow_counter_out { 2200 uint8_t cmd_status; 2201 uint8_t cmd_reserved0[3]; 2202 uint32_t cmd_syndrome; 2203 uint8_t cmd_reserved1[2]; 2204 uint16_t cmd_flow_counter_id; 2205 uint8_t cmd_reserved2[4]; 2206 } __packed __aligned(4); 2207 2208 struct mcx_wq_doorbell { 2209 uint32_t db_recv_counter; 2210 uint32_t db_send_counter; 2211 } __packed __aligned(8); 2212 2213 struct mcx_dmamem { 2214 bus_dmamap_t mxm_map; 2215 bus_dma_segment_t mxm_seg; 2216 int mxm_nsegs; 2217 size_t mxm_size; 2218 caddr_t mxm_kva; 2219 }; 2220 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2221 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2222 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2223 #define MCX_DMA_OFF(_mxm, _off) ((void *)((_mxm)->mxm_kva + (_off))) 2224 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2225 2226 struct mcx_hwmem { 2227 bus_dmamap_t mhm_map; 2228 bus_dma_segment_t *mhm_segs; 2229 unsigned int mhm_seg_count; 2230 unsigned int mhm_npages; 2231 }; 2232 2233 struct mcx_slot { 2234 bus_dmamap_t ms_map; 2235 struct mbuf *ms_m; 2236 }; 2237 2238 struct mcx_eq { 2239 int eq_n; 2240 uint32_t eq_cons; 2241 struct mcx_dmamem eq_mem; 2242 }; 2243 2244 struct mcx_cq { 2245 int cq_n; 2246 struct mcx_dmamem cq_mem; 2247 bus_addr_t cq_doorbell; 2248 uint32_t cq_cons; 2249 uint32_t cq_count; 2250 }; 2251 2252 struct mcx_calibration { 2253 uint64_t c_timestamp; /* previous mcx chip time */ 2254 uint64_t c_uptime; /* previous kernel nanouptime */ 2255 uint64_t c_tbase; /* mcx chip time */ 2256 uint64_t c_ubase; /* kernel nanouptime */ 2257 uint64_t c_ratio; 2258 }; 2259 2260 #define MCX_CALIBRATE_FIRST 2 2261 #define MCX_CALIBRATE_NORMAL 32 2262 2263 struct mcx_rx { 2264 struct mcx_softc *rx_softc; 2265 struct ifiqueue *rx_ifiq; 2266 2267 int rx_rqn; 2268 struct mcx_dmamem rx_rq_mem; 2269 struct mcx_slot *rx_slots; 2270 bus_addr_t rx_doorbell; 2271 2272 uint32_t rx_prod; 2273 struct timeout rx_refill; 2274 struct if_rxring rx_rxr; 2275 } __aligned(64); 2276 2277 struct mcx_tx { 2278 struct mcx_softc *tx_softc; 2279 struct ifqueue *tx_ifq; 2280 2281 int tx_uar; 2282 int tx_sqn; 2283 struct mcx_dmamem tx_sq_mem; 2284 struct mcx_slot *tx_slots; 2285 bus_addr_t tx_doorbell; 2286 int tx_bf_offset; 2287 2288 uint32_t tx_cons; 2289 uint32_t tx_prod; 2290 } __aligned(64); 2291 2292 struct mcx_queues { 2293 char q_name[16]; 2294 void *q_ihc; 2295 struct mcx_softc *q_sc; 2296 int q_uar; 2297 int q_index; 2298 struct mcx_rx q_rx; 2299 struct mcx_tx q_tx; 2300 struct mcx_cq q_cq; 2301 struct mcx_eq q_eq; 2302 #if NKSTAT > 0 2303 struct kstat *q_kstat; 2304 #endif 2305 }; 2306 2307 struct mcx_flow_group { 2308 int g_id; 2309 int g_table; 2310 int g_start; 2311 int g_size; 2312 }; 2313 2314 #define MCX_FLOW_GROUP_PROMISC 0 2315 #define MCX_FLOW_GROUP_ALLMULTI 1 2316 #define MCX_FLOW_GROUP_MAC 2 2317 #define MCX_FLOW_GROUP_RSS_L4 3 2318 #define MCX_FLOW_GROUP_RSS_L3 4 2319 #define MCX_FLOW_GROUP_RSS_NONE 5 2320 #define MCX_NUM_FLOW_GROUPS 6 2321 2322 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2323 MCX_TIR_CTX_HASH_SEL_DST_IP 2324 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2325 MCX_TIR_CTX_HASH_SEL_DPORT 2326 2327 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2328 MCX_TIR_CTX_HASH_SEL_IPV4 2329 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2330 MCX_TIR_CTX_HASH_SEL_IPV6 2331 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2332 MCX_TIR_CTX_HASH_SEL_IPV4 2333 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2334 MCX_TIR_CTX_HASH_SEL_IPV6 2335 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2336 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2337 2338 /* 2339 * There are a few different pieces involved in configuring RSS. 2340 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2341 * different rx queues based on a hash value. We only create one, because 2342 * we want to scatter any traffic we can apply RSS to across all our rx 2343 * queues. Anything else will only be delivered to the first rx queue, 2344 * which doesn't require an RQT. 2345 * 2346 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2347 * queue or an RQT, and in the latter case, specifies the set of fields 2348 * hashed, the hash function, and the hash key. We need one of these for each 2349 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2350 * and one for non-RSS traffic. 2351 * 2352 * Flow tables hold flow table entries in sequence. The first entry that 2353 * matches a packet is applied, sending the packet to either another flow 2354 * table or a TIR. We use one flow table to select packets based on 2355 * destination MAC address, and a second to apply RSS. The entries in the 2356 * first table send matching packets to the second, and the entries in the 2357 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2358 * 2359 * The flow table entry that delivers packets to an RSS TIR must include match 2360 * criteria that ensure packets delivered to the TIR include all the fields 2361 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2362 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2363 * each TIR. 2364 * 2365 * All of this is a lot more flexible than we need, and we can describe most 2366 * of the stuff we need with a simple array. 2367 * 2368 * An RSS config creates a TIR with hashing enabled on a set of fields, 2369 * pointing to either the first rx queue or the RQT containing all the rx 2370 * queues, and a flow table entry that matches on an ether type and 2371 * optionally an ip proto, that delivers packets to the TIR. 2372 */ 2373 static struct mcx_rss_rule { 2374 int hash_sel; 2375 int flow_group; 2376 int ethertype; 2377 int ip_proto; 2378 } mcx_rss_config[] = { 2379 /* udp and tcp for v4/v6 */ 2380 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2381 ETHERTYPE_IP, IPPROTO_TCP }, 2382 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2383 ETHERTYPE_IPV6, IPPROTO_TCP }, 2384 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2385 ETHERTYPE_IP, IPPROTO_UDP }, 2386 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2387 ETHERTYPE_IPV6, IPPROTO_UDP }, 2388 2389 /* other v4/v6 */ 2390 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2391 ETHERTYPE_IP, 0 }, 2392 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2393 ETHERTYPE_IPV6, 0 }, 2394 2395 /* non v4/v6 */ 2396 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2397 }; 2398 2399 struct mcx_softc { 2400 struct device sc_dev; 2401 struct arpcom sc_ac; 2402 struct ifmedia sc_media; 2403 uint64_t sc_media_status; 2404 uint64_t sc_media_active; 2405 2406 pci_chipset_tag_t sc_pc; 2407 pci_intr_handle_t sc_ih; 2408 void *sc_ihc; 2409 pcitag_t sc_tag; 2410 2411 bus_dma_tag_t sc_dmat; 2412 bus_space_tag_t sc_memt; 2413 bus_space_handle_t sc_memh; 2414 bus_size_t sc_mems; 2415 2416 struct mcx_dmamem sc_cmdq_mem; 2417 unsigned int sc_cmdq_mask; 2418 unsigned int sc_cmdq_size; 2419 2420 unsigned int sc_cmdq_token; 2421 2422 struct mcx_hwmem sc_boot_pages; 2423 struct mcx_hwmem sc_init_pages; 2424 struct mcx_hwmem sc_regular_pages; 2425 2426 int sc_uar; 2427 int sc_pd; 2428 int sc_tdomain; 2429 uint32_t sc_lkey; 2430 int sc_tis; 2431 int sc_tir[nitems(mcx_rss_config)]; 2432 int sc_rqt; 2433 2434 struct mcx_dmamem sc_doorbell_mem; 2435 2436 struct mcx_eq sc_admin_eq; 2437 struct mcx_eq sc_queue_eq; 2438 2439 int sc_hardmtu; 2440 int sc_rxbufsz; 2441 2442 int sc_bf_size; 2443 int sc_max_rqt_size; 2444 2445 struct task sc_port_change; 2446 2447 int sc_mac_flow_table_id; 2448 int sc_rss_flow_table_id; 2449 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2450 int sc_promisc_flow_enabled; 2451 int sc_allmulti_flow_enabled; 2452 int sc_mcast_flow_base; 2453 int sc_extra_mcast; 2454 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2455 2456 struct mcx_calibration sc_calibration[2]; 2457 unsigned int sc_calibration_gen; 2458 struct timeout sc_calibrate; 2459 uint32_t sc_mhz; 2460 uint32_t sc_khz; 2461 2462 struct mcx_queues sc_queues[MCX_MAX_QUEUES]; 2463 unsigned int sc_nqueues; 2464 2465 #if NKSTAT > 0 2466 struct kstat *sc_kstat_ieee8023; 2467 struct kstat *sc_kstat_rfc2863; 2468 struct kstat *sc_kstat_rfc2819; 2469 struct kstat *sc_kstat_rfc3635; 2470 unsigned int sc_kstat_mtmp_count; 2471 struct kstat **sc_kstat_mtmp; 2472 #endif 2473 2474 struct timecounter sc_timecounter; 2475 }; 2476 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2477 2478 static int mcx_match(struct device *, void *, void *); 2479 static void mcx_attach(struct device *, struct device *, void *); 2480 2481 #if NKSTAT > 0 2482 static void mcx_kstat_attach(struct mcx_softc *); 2483 #endif 2484 2485 static void mcx_timecounter_attach(struct mcx_softc *); 2486 2487 static int mcx_version(struct mcx_softc *); 2488 static int mcx_init_wait(struct mcx_softc *); 2489 static int mcx_enable_hca(struct mcx_softc *); 2490 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2491 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2492 int); 2493 static int mcx_issi(struct mcx_softc *); 2494 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2495 static int mcx_hca_max_caps(struct mcx_softc *); 2496 static int mcx_hca_set_caps(struct mcx_softc *); 2497 static int mcx_init_hca(struct mcx_softc *); 2498 static int mcx_set_driver_version(struct mcx_softc *); 2499 static int mcx_iff(struct mcx_softc *); 2500 static int mcx_alloc_uar(struct mcx_softc *, int *); 2501 static int mcx_alloc_pd(struct mcx_softc *); 2502 static int mcx_alloc_tdomain(struct mcx_softc *); 2503 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2504 uint64_t, int); 2505 static int mcx_query_nic_vport_context(struct mcx_softc *); 2506 static int mcx_query_special_contexts(struct mcx_softc *); 2507 static int mcx_set_port_mtu(struct mcx_softc *, int); 2508 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2509 int); 2510 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2511 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2512 int); 2513 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2514 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2515 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2516 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2517 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2518 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2519 int *); 2520 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2521 int *); 2522 static int mcx_destroy_tir(struct mcx_softc *, int); 2523 static int mcx_create_tis(struct mcx_softc *, int *); 2524 static int mcx_destroy_tis(struct mcx_softc *, int); 2525 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2526 static int mcx_destroy_rqt(struct mcx_softc *, int); 2527 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2528 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2529 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2530 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2531 int, int, struct mcx_flow_match *); 2532 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2533 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2534 uint8_t *, uint32_t); 2535 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2536 int, int, uint32_t); 2537 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2538 2539 #if NKSTAT > 0 2540 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *); 2541 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *); 2542 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *); 2543 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *); 2544 #endif 2545 2546 #if 0 2547 static int mcx_dump_flow_table(struct mcx_softc *, int); 2548 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2549 static int mcx_dump_flow_group(struct mcx_softc *, int); 2550 #endif 2551 2552 2553 /* 2554 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2555 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2556 */ 2557 static void mcx_refill(void *); 2558 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2559 struct mcx_cq_entry *, struct mbuf_list *, 2560 const struct mcx_calibration *); 2561 static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2562 struct mcx_cq_entry *); 2563 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2564 struct mcx_cq *); 2565 2566 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2567 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2568 static int mcx_admin_intr(void *); 2569 static int mcx_cq_intr(void *); 2570 2571 static int mcx_up(struct mcx_softc *); 2572 static void mcx_down(struct mcx_softc *); 2573 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2574 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2575 static void mcx_start(struct ifqueue *); 2576 static void mcx_watchdog(struct ifnet *); 2577 static void mcx_media_add_types(struct mcx_softc *); 2578 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2579 static int mcx_media_change(struct ifnet *); 2580 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2581 static void mcx_port_change(void *); 2582 2583 static void mcx_calibrate_first(struct mcx_softc *); 2584 static void mcx_calibrate(void *); 2585 2586 static inline uint32_t 2587 mcx_rd(struct mcx_softc *, bus_size_t); 2588 static inline void 2589 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2590 static inline void 2591 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2592 2593 static uint64_t mcx_timer(struct mcx_softc *); 2594 2595 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2596 bus_size_t, u_int align); 2597 static void mcx_dmamem_zero(struct mcx_dmamem *); 2598 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2599 2600 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2601 unsigned int); 2602 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2603 2604 struct cfdriver mcx_cd = { 2605 NULL, 2606 "mcx", 2607 DV_IFNET, 2608 }; 2609 2610 struct cfattach mcx_ca = { 2611 sizeof(struct mcx_softc), 2612 mcx_match, 2613 mcx_attach, 2614 }; 2615 2616 static const struct pci_matchid mcx_devices[] = { 2617 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2618 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2619 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2620 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2621 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2622 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2623 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2624 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2625 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 }, 2626 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2627 }; 2628 2629 struct mcx_eth_proto_capability { 2630 uint64_t cap_media; 2631 uint64_t cap_baudrate; 2632 }; 2633 2634 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2635 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2636 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2637 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2638 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2639 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2640 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2641 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2642 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2643 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2644 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2645 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2646 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2647 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2648 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2649 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2650 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2651 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2652 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2653 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2654 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2655 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2656 }; 2657 2658 static int 2659 mcx_get_id(uint32_t val) 2660 { 2661 return betoh32(val) & 0x00ffffff; 2662 } 2663 2664 static int 2665 mcx_match(struct device *parent, void *match, void *aux) 2666 { 2667 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2668 } 2669 2670 void 2671 mcx_attach(struct device *parent, struct device *self, void *aux) 2672 { 2673 struct mcx_softc *sc = (struct mcx_softc *)self; 2674 struct ifnet *ifp = &sc->sc_ac.ac_if; 2675 struct pci_attach_args *pa = aux; 2676 pcireg_t memtype; 2677 uint32_t r; 2678 unsigned int cq_stride; 2679 unsigned int cq_size; 2680 const char *intrstr; 2681 int i, msix; 2682 2683 sc->sc_pc = pa->pa_pc; 2684 sc->sc_tag = pa->pa_tag; 2685 sc->sc_dmat = pa->pa_dmat; 2686 2687 /* Map the PCI memory space */ 2688 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2689 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2690 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2691 NULL, &sc->sc_mems, 0)) { 2692 printf(": unable to map register memory\n"); 2693 return; 2694 } 2695 2696 if (mcx_version(sc) != 0) { 2697 /* error printed by mcx_version */ 2698 goto unmap; 2699 } 2700 2701 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2702 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2703 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2704 if (cq_size > MCX_MAX_CQE) { 2705 printf(", command queue size overflow %u\n", cq_size); 2706 goto unmap; 2707 } 2708 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2709 printf(", command queue entry size underflow %u\n", cq_stride); 2710 goto unmap; 2711 } 2712 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2713 printf(", command queue page overflow\n"); 2714 goto unmap; 2715 } 2716 2717 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2718 MCX_PAGE_SIZE) != 0) { 2719 printf(", unable to allocate doorbell memory\n"); 2720 goto unmap; 2721 } 2722 2723 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2724 MCX_PAGE_SIZE) != 0) { 2725 printf(", unable to allocate command queue\n"); 2726 goto dbfree; 2727 } 2728 2729 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2730 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2731 BUS_SPACE_BARRIER_WRITE); 2732 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2733 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2734 BUS_SPACE_BARRIER_WRITE); 2735 2736 if (mcx_init_wait(sc) != 0) { 2737 printf(", timeout waiting for init\n"); 2738 goto cqfree; 2739 } 2740 2741 sc->sc_cmdq_mask = cq_size - 1; 2742 sc->sc_cmdq_size = cq_stride; 2743 2744 if (mcx_enable_hca(sc) != 0) { 2745 /* error printed by mcx_enable_hca */ 2746 goto cqfree; 2747 } 2748 2749 if (mcx_issi(sc) != 0) { 2750 /* error printed by mcx_issi */ 2751 goto teardown; 2752 } 2753 2754 if (mcx_pages(sc, &sc->sc_boot_pages, 2755 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2756 /* error printed by mcx_pages */ 2757 goto teardown; 2758 } 2759 2760 if (mcx_hca_max_caps(sc) != 0) { 2761 /* error printed by mcx_hca_max_caps */ 2762 goto teardown; 2763 } 2764 2765 if (mcx_hca_set_caps(sc) != 0) { 2766 /* error printed by mcx_hca_set_caps */ 2767 goto teardown; 2768 } 2769 2770 if (mcx_pages(sc, &sc->sc_init_pages, 2771 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2772 /* error printed by mcx_pages */ 2773 goto teardown; 2774 } 2775 2776 if (mcx_init_hca(sc) != 0) { 2777 /* error printed by mcx_init_hca */ 2778 goto teardown; 2779 } 2780 2781 if (mcx_pages(sc, &sc->sc_regular_pages, 2782 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2783 /* error printed by mcx_pages */ 2784 goto teardown; 2785 } 2786 2787 /* apparently not necessary? */ 2788 if (mcx_set_driver_version(sc) != 0) { 2789 /* error printed by mcx_set_driver_version */ 2790 goto teardown; 2791 } 2792 2793 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2794 /* error printed by mcx_iff? */ 2795 goto teardown; 2796 } 2797 2798 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2799 /* error printed by mcx_alloc_uar */ 2800 goto teardown; 2801 } 2802 2803 if (mcx_alloc_pd(sc) != 0) { 2804 /* error printed by mcx_alloc_pd */ 2805 goto teardown; 2806 } 2807 2808 if (mcx_alloc_tdomain(sc) != 0) { 2809 /* error printed by mcx_alloc_tdomain */ 2810 goto teardown; 2811 } 2812 2813 /* 2814 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2815 * mellanox support tells me legacy interrupts are not supported, 2816 * so we're stuck with just msi-x. 2817 */ 2818 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2819 printf(": unable to map interrupt\n"); 2820 goto teardown; 2821 } 2822 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2823 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2824 IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc)); 2825 if (sc->sc_ihc == NULL) { 2826 printf(": unable to establish interrupt"); 2827 if (intrstr != NULL) 2828 printf(" at %s", intrstr); 2829 printf("\n"); 2830 goto teardown; 2831 } 2832 2833 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2834 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2835 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2836 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2837 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2838 /* error printed by mcx_create_eq */ 2839 goto teardown; 2840 } 2841 2842 if (mcx_query_nic_vport_context(sc) != 0) { 2843 /* error printed by mcx_query_nic_vport_context */ 2844 goto teardown; 2845 } 2846 2847 if (mcx_query_special_contexts(sc) != 0) { 2848 /* error printed by mcx_query_special_contexts */ 2849 goto teardown; 2850 } 2851 2852 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2853 /* error printed by mcx_set_port_mtu */ 2854 goto teardown; 2855 } 2856 2857 printf(", %s, address %s\n", intrstr, 2858 ether_sprintf(sc->sc_ac.ac_enaddr)); 2859 2860 msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); 2861 sc->sc_nqueues = 1; 2862 2863 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2864 ifp->if_softc = sc; 2865 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2866 ifp->if_xflags = IFXF_MPSAFE; 2867 ifp->if_ioctl = mcx_ioctl; 2868 ifp->if_qstart = mcx_start; 2869 ifp->if_watchdog = mcx_watchdog; 2870 ifp->if_hardmtu = sc->sc_hardmtu; 2871 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 | 2872 IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 | 2873 IFCAP_CSUM_TCPv6; 2874 #if NVLAN > 0 2875 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2876 #endif 2877 ifq_set_maxlen(&ifp->if_snd, 1024); 2878 2879 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2880 mcx_media_status); 2881 mcx_media_add_types(sc); 2882 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2883 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2884 2885 if_attach(ifp); 2886 ether_ifattach(ifp); 2887 2888 if_attach_iqueues(ifp, sc->sc_nqueues); 2889 if_attach_queues(ifp, sc->sc_nqueues); 2890 for (i = 0; i < sc->sc_nqueues; i++) { 2891 struct ifiqueue *ifiq = ifp->if_iqs[i]; 2892 struct ifqueue *ifq = ifp->if_ifqs[i]; 2893 struct mcx_queues *q = &sc->sc_queues[i]; 2894 struct mcx_rx *rx = &q->q_rx; 2895 struct mcx_tx *tx = &q->q_tx; 2896 pci_intr_handle_t ih; 2897 int vec; 2898 2899 vec = i + 1; 2900 q->q_sc = sc; 2901 q->q_index = i; 2902 2903 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 2904 printf("%s: unable to alloc uar %d\n", 2905 DEVNAME(sc), i); 2906 goto teardown; 2907 } 2908 2909 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 2910 printf("%s: unable to create event queue %d\n", 2911 DEVNAME(sc), i); 2912 goto teardown; 2913 } 2914 2915 rx->rx_softc = sc; 2916 rx->rx_ifiq = ifiq; 2917 timeout_set(&rx->rx_refill, mcx_refill, rx); 2918 ifiq->ifiq_softc = rx; 2919 2920 tx->tx_softc = sc; 2921 tx->tx_ifq = ifq; 2922 ifq->ifq_softc = tx; 2923 2924 if (pci_intr_map_msix(pa, vec, &ih) != 0) { 2925 printf("%s: unable to map queue interrupt %d\n", 2926 DEVNAME(sc), i); 2927 goto teardown; 2928 } 2929 snprintf(q->q_name, sizeof(q->q_name), "%s:%d", 2930 DEVNAME(sc), i); 2931 q->q_ihc = pci_intr_establish(sc->sc_pc, ih, 2932 IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); 2933 } 2934 2935 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2936 2937 task_set(&sc->sc_port_change, mcx_port_change, sc); 2938 mcx_port_change(sc); 2939 2940 sc->sc_mac_flow_table_id = -1; 2941 sc->sc_rss_flow_table_id = -1; 2942 sc->sc_rqt = -1; 2943 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 2944 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 2945 mfg->g_id = -1; 2946 mfg->g_table = -1; 2947 mfg->g_size = 0; 2948 mfg->g_start = 0; 2949 } 2950 sc->sc_extra_mcast = 0; 2951 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 2952 2953 #if NKSTAT > 0 2954 mcx_kstat_attach(sc); 2955 #endif 2956 mcx_timecounter_attach(sc); 2957 return; 2958 2959 teardown: 2960 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 2961 /* error printed by mcx_teardown_hca, and we're already unwinding */ 2962 cqfree: 2963 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2964 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2965 BUS_SPACE_BARRIER_WRITE); 2966 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 2967 MCX_CMDQ_INTERFACE_DISABLED); 2968 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 2969 BUS_SPACE_BARRIER_WRITE); 2970 2971 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 2972 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2973 BUS_SPACE_BARRIER_WRITE); 2974 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 2975 2976 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 2977 dbfree: 2978 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 2979 unmap: 2980 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 2981 sc->sc_mems = 0; 2982 } 2983 2984 static int 2985 mcx_version(struct mcx_softc *sc) 2986 { 2987 uint32_t fw0, fw1; 2988 uint16_t cmdif; 2989 2990 fw0 = mcx_rd(sc, MCX_FW_VER); 2991 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 2992 2993 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 2994 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 2995 2996 cmdif = MCX_CMDIF(fw1); 2997 if (cmdif != MCX_CMD_IF_SUPPORTED) { 2998 printf(", unsupported command interface %u\n", cmdif); 2999 return (-1); 3000 } 3001 3002 return (0); 3003 } 3004 3005 static int 3006 mcx_init_wait(struct mcx_softc *sc) 3007 { 3008 unsigned int i; 3009 uint32_t r; 3010 3011 for (i = 0; i < 2000; i++) { 3012 r = mcx_rd(sc, MCX_STATE); 3013 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 3014 return (0); 3015 3016 delay(1000); 3017 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 3018 BUS_SPACE_BARRIER_READ); 3019 } 3020 3021 return (-1); 3022 } 3023 3024 static uint8_t 3025 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3026 unsigned int msec) 3027 { 3028 unsigned int i; 3029 3030 for (i = 0; i < msec; i++) { 3031 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3032 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3033 3034 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 3035 MCX_CQ_STATUS_OWN_SW) 3036 return (0); 3037 3038 delay(1000); 3039 } 3040 3041 return (ETIMEDOUT); 3042 } 3043 3044 static uint32_t 3045 mcx_mix_u64(uint32_t xor, uint64_t u64) 3046 { 3047 xor ^= u64 >> 32; 3048 xor ^= u64; 3049 3050 return (xor); 3051 } 3052 3053 static uint32_t 3054 mcx_mix_u32(uint32_t xor, uint32_t u32) 3055 { 3056 xor ^= u32; 3057 3058 return (xor); 3059 } 3060 3061 static uint32_t 3062 mcx_mix_u8(uint32_t xor, uint8_t u8) 3063 { 3064 xor ^= u8; 3065 3066 return (xor); 3067 } 3068 3069 static uint8_t 3070 mcx_mix_done(uint32_t xor) 3071 { 3072 xor ^= xor >> 16; 3073 xor ^= xor >> 8; 3074 3075 return (xor); 3076 } 3077 3078 static uint8_t 3079 mcx_xor(const void *buf, size_t len) 3080 { 3081 const uint32_t *dwords = buf; 3082 uint32_t xor = 0xff; 3083 size_t i; 3084 3085 len /= sizeof(*dwords); 3086 3087 for (i = 0; i < len; i++) 3088 xor ^= dwords[i]; 3089 3090 return (mcx_mix_done(xor)); 3091 } 3092 3093 static uint8_t 3094 mcx_cmdq_token(struct mcx_softc *sc) 3095 { 3096 uint8_t token; 3097 3098 do { 3099 token = ++sc->sc_cmdq_token; 3100 } while (token == 0); 3101 3102 return (token); 3103 } 3104 3105 static void 3106 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3107 uint32_t ilen, uint32_t olen, uint8_t token) 3108 { 3109 memset(cqe, 0, sc->sc_cmdq_size); 3110 3111 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 3112 htobem32(&cqe->cq_input_length, ilen); 3113 htobem32(&cqe->cq_output_length, olen); 3114 cqe->cq_token = token; 3115 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 3116 } 3117 3118 static void 3119 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 3120 { 3121 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 3122 } 3123 3124 static int 3125 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 3126 { 3127 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 3128 return (0); 3129 } 3130 3131 static void * 3132 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 3133 { 3134 return (&cqe->cq_input_data); 3135 } 3136 3137 static void * 3138 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3139 { 3140 return (&cqe->cq_output_data); 3141 } 3142 3143 static void 3144 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3145 unsigned int slot) 3146 { 3147 mcx_cmdq_sign(cqe); 3148 3149 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3150 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3151 3152 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3153 mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t), 3154 BUS_SPACE_BARRIER_WRITE); 3155 } 3156 3157 static int 3158 mcx_enable_hca(struct mcx_softc *sc) 3159 { 3160 struct mcx_cmdq_entry *cqe; 3161 struct mcx_cmd_enable_hca_in *in; 3162 struct mcx_cmd_enable_hca_out *out; 3163 int error; 3164 uint8_t status; 3165 3166 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3167 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3168 3169 in = mcx_cmdq_in(cqe); 3170 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3171 in->cmd_op_mod = htobe16(0); 3172 in->cmd_function_id = htobe16(0); 3173 3174 mcx_cmdq_post(sc, cqe, 0); 3175 3176 error = mcx_cmdq_poll(sc, cqe, 1000); 3177 if (error != 0) { 3178 printf(", hca enable timeout\n"); 3179 return (-1); 3180 } 3181 if (mcx_cmdq_verify(cqe) != 0) { 3182 printf(", hca enable command corrupt\n"); 3183 return (-1); 3184 } 3185 3186 status = cqe->cq_output_data[0]; 3187 if (status != MCX_CQ_STATUS_OK) { 3188 printf(", hca enable failed (%x)\n", status); 3189 return (-1); 3190 } 3191 3192 return (0); 3193 } 3194 3195 static int 3196 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3197 { 3198 struct mcx_cmdq_entry *cqe; 3199 struct mcx_cmd_teardown_hca_in *in; 3200 struct mcx_cmd_teardown_hca_out *out; 3201 int error; 3202 uint8_t status; 3203 3204 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3205 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3206 3207 in = mcx_cmdq_in(cqe); 3208 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3209 in->cmd_op_mod = htobe16(0); 3210 in->cmd_profile = profile; 3211 3212 mcx_cmdq_post(sc, cqe, 0); 3213 3214 error = mcx_cmdq_poll(sc, cqe, 1000); 3215 if (error != 0) { 3216 printf(", hca teardown timeout\n"); 3217 return (-1); 3218 } 3219 if (mcx_cmdq_verify(cqe) != 0) { 3220 printf(", hca teardown command corrupt\n"); 3221 return (-1); 3222 } 3223 3224 status = cqe->cq_output_data[0]; 3225 if (status != MCX_CQ_STATUS_OK) { 3226 printf(", hca teardown failed (%x)\n", status); 3227 return (-1); 3228 } 3229 3230 return (0); 3231 } 3232 3233 static int 3234 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3235 unsigned int nmb, uint64_t *ptr, uint8_t token) 3236 { 3237 caddr_t kva; 3238 uint64_t dva; 3239 int i; 3240 int error; 3241 3242 error = mcx_dmamem_alloc(sc, mxm, 3243 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3244 if (error != 0) 3245 return (error); 3246 3247 mcx_dmamem_zero(mxm); 3248 3249 dva = MCX_DMA_DVA(mxm); 3250 kva = MCX_DMA_KVA(mxm); 3251 for (i = 0; i < nmb; i++) { 3252 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3253 3254 /* patch the cqe or mbox pointing at this one */ 3255 htobem64(ptr, dva); 3256 3257 /* fill in this mbox */ 3258 htobem32(&mbox->mb_block_number, i); 3259 mbox->mb_token = token; 3260 3261 /* move to the next one */ 3262 ptr = &mbox->mb_next_ptr; 3263 3264 dva += MCX_CMDQ_MAILBOX_SIZE; 3265 kva += MCX_CMDQ_MAILBOX_SIZE; 3266 } 3267 3268 return (0); 3269 } 3270 3271 static uint32_t 3272 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3273 { 3274 uint32_t xor = 0xff; 3275 3276 /* only 3 fields get set, so mix them directly */ 3277 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3278 xor = mcx_mix_u32(xor, mb->mb_block_number); 3279 xor = mcx_mix_u8(xor, mb->mb_token); 3280 3281 return (mcx_mix_done(xor)); 3282 } 3283 3284 static void 3285 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3286 { 3287 caddr_t kva; 3288 int i; 3289 3290 kva = MCX_DMA_KVA(mxm); 3291 3292 for (i = 0; i < nmb; i++) { 3293 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3294 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3295 mb->mb_ctrl_signature = sig; 3296 mb->mb_signature = sig ^ 3297 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3298 3299 kva += MCX_CMDQ_MAILBOX_SIZE; 3300 } 3301 } 3302 3303 static void 3304 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3305 { 3306 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3307 0, MCX_DMA_LEN(mxm), ops); 3308 } 3309 3310 static struct mcx_cmdq_mailbox * 3311 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3312 { 3313 caddr_t kva; 3314 3315 kva = MCX_DMA_KVA(mxm); 3316 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3317 3318 return ((struct mcx_cmdq_mailbox *)kva); 3319 } 3320 3321 static inline void * 3322 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3323 { 3324 return (&mb->mb_data); 3325 } 3326 3327 static void 3328 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3329 void *b, size_t len) 3330 { 3331 caddr_t buf = b; 3332 struct mcx_cmdq_mailbox *mb; 3333 int i; 3334 3335 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3336 for (i = 0; i < nmb; i++) { 3337 3338 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 3339 3340 if (sizeof(mb->mb_data) >= len) 3341 break; 3342 3343 buf += sizeof(mb->mb_data); 3344 len -= sizeof(mb->mb_data); 3345 mb++; 3346 } 3347 } 3348 3349 static void 3350 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3351 struct mcx_dmamem *buf) 3352 { 3353 uint64_t *pas; 3354 int mbox, mbox_pages, i; 3355 3356 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3357 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3358 3359 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3360 pas += (offset / sizeof(*pas)); 3361 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3362 for (i = 0; i < npages; i++) { 3363 if (i == mbox_pages) { 3364 mbox++; 3365 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3366 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3367 } 3368 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3369 pas++; 3370 } 3371 } 3372 3373 static void 3374 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3375 { 3376 caddr_t buf = b; 3377 struct mcx_cmdq_mailbox *mb; 3378 int i; 3379 3380 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3381 for (i = 0; i < nmb; i++) { 3382 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 3383 3384 if (sizeof(mb->mb_data) >= len) 3385 break; 3386 3387 buf += sizeof(mb->mb_data); 3388 len -= sizeof(mb->mb_data); 3389 mb++; 3390 } 3391 } 3392 3393 static void 3394 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3395 { 3396 mcx_dmamem_free(sc, mxm); 3397 } 3398 3399 #if 0 3400 static void 3401 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3402 { 3403 unsigned int i; 3404 3405 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3406 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 3407 3408 printf(", idata "); 3409 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3410 printf("%02x", cqe->cq_input_data[i]); 3411 3412 printf(", odata "); 3413 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3414 printf("%02x", cqe->cq_output_data[i]); 3415 3416 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3417 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 3418 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3419 } 3420 3421 static void 3422 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3423 { 3424 int i, j; 3425 uint8_t *d; 3426 3427 for (i = 0; i < num; i++) { 3428 struct mcx_cmdq_mailbox *mbox; 3429 mbox = mcx_cq_mbox(mboxes, i); 3430 3431 d = mcx_cq_mbox_data(mbox); 3432 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3433 if (j != 0 && (j % 16 == 0)) 3434 printf("\n"); 3435 printf("%.2x ", d[j]); 3436 } 3437 } 3438 } 3439 #endif 3440 3441 static int 3442 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3443 int len) 3444 { 3445 struct mcx_dmamem mxm; 3446 struct mcx_cmdq_entry *cqe; 3447 struct mcx_cmd_access_reg_in *in; 3448 struct mcx_cmd_access_reg_out *out; 3449 uint8_t token = mcx_cmdq_token(sc); 3450 int error, nmb; 3451 3452 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3453 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3454 token); 3455 3456 in = mcx_cmdq_in(cqe); 3457 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3458 in->cmd_op_mod = htobe16(op); 3459 in->cmd_register_id = htobe16(reg); 3460 3461 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3462 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3463 &cqe->cq_output_ptr, token) != 0) { 3464 printf(", unable to allocate access reg mailboxen\n"); 3465 return (-1); 3466 } 3467 cqe->cq_input_ptr = cqe->cq_output_ptr; 3468 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3469 mcx_cmdq_mboxes_sign(&mxm, nmb); 3470 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3471 3472 mcx_cmdq_post(sc, cqe, 0); 3473 error = mcx_cmdq_poll(sc, cqe, 1000); 3474 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3475 3476 if (error != 0) { 3477 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3478 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3479 goto free; 3480 } 3481 error = mcx_cmdq_verify(cqe); 3482 if (error != 0) { 3483 printf("%s: access reg (%s %x) reply corrupt\n", 3484 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3485 reg); 3486 goto free; 3487 } 3488 3489 out = mcx_cmdq_out(cqe); 3490 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3491 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3492 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3493 reg, out->cmd_status, betoh32(out->cmd_syndrome)); 3494 error = -1; 3495 goto free; 3496 } 3497 3498 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3499 free: 3500 mcx_dmamem_free(sc, &mxm); 3501 3502 return (error); 3503 } 3504 3505 static int 3506 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3507 unsigned int slot) 3508 { 3509 struct mcx_cmd_set_issi_in *in; 3510 struct mcx_cmd_set_issi_out *out; 3511 uint8_t status; 3512 3513 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3514 3515 in = mcx_cmdq_in(cqe); 3516 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3517 in->cmd_op_mod = htobe16(0); 3518 in->cmd_current_issi = htobe16(MCX_ISSI); 3519 3520 mcx_cmdq_post(sc, cqe, slot); 3521 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3522 return (-1); 3523 if (mcx_cmdq_verify(cqe) != 0) 3524 return (-1); 3525 3526 status = cqe->cq_output_data[0]; 3527 if (status != MCX_CQ_STATUS_OK) 3528 return (-1); 3529 3530 return (0); 3531 } 3532 3533 static int 3534 mcx_issi(struct mcx_softc *sc) 3535 { 3536 struct mcx_dmamem mxm; 3537 struct mcx_cmdq_entry *cqe; 3538 struct mcx_cmd_query_issi_in *in; 3539 struct mcx_cmd_query_issi_il_out *out; 3540 struct mcx_cmd_query_issi_mb_out *mb; 3541 uint8_t token = mcx_cmdq_token(sc); 3542 uint8_t status; 3543 int error; 3544 3545 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3546 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3547 3548 in = mcx_cmdq_in(cqe); 3549 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3550 in->cmd_op_mod = htobe16(0); 3551 3552 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3553 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3554 &cqe->cq_output_ptr, token) != 0) { 3555 printf(", unable to allocate query issi mailbox\n"); 3556 return (-1); 3557 } 3558 mcx_cmdq_mboxes_sign(&mxm, 1); 3559 3560 mcx_cmdq_post(sc, cqe, 0); 3561 error = mcx_cmdq_poll(sc, cqe, 1000); 3562 if (error != 0) { 3563 printf(", query issi timeout\n"); 3564 goto free; 3565 } 3566 error = mcx_cmdq_verify(cqe); 3567 if (error != 0) { 3568 printf(", query issi reply corrupt\n"); 3569 goto free; 3570 } 3571 3572 status = cqe->cq_output_data[0]; 3573 switch (status) { 3574 case MCX_CQ_STATUS_OK: 3575 break; 3576 case MCX_CQ_STATUS_BAD_OPCODE: 3577 /* use ISSI 0 */ 3578 goto free; 3579 default: 3580 printf(", query issi failed (%x)\n", status); 3581 error = -1; 3582 goto free; 3583 } 3584 3585 out = mcx_cmdq_out(cqe); 3586 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3587 /* use ISSI 1 */ 3588 goto free; 3589 } 3590 3591 /* don't need to read cqe anymore, can be used for SET ISSI */ 3592 3593 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3594 CTASSERT(MCX_ISSI < NBBY); 3595 /* XXX math is hard */ 3596 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3597 /* use ISSI 0 */ 3598 goto free; 3599 } 3600 3601 if (mcx_set_issi(sc, cqe, 0) != 0) { 3602 /* ignore the error, just use ISSI 0 */ 3603 } else { 3604 /* use ISSI 1 */ 3605 } 3606 3607 free: 3608 mcx_cq_mboxes_free(sc, &mxm); 3609 return (error); 3610 } 3611 3612 static int 3613 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3614 int32_t *npages, uint16_t *func_id) 3615 { 3616 struct mcx_cmdq_entry *cqe; 3617 struct mcx_cmd_query_pages_in *in; 3618 struct mcx_cmd_query_pages_out *out; 3619 3620 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3621 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3622 3623 in = mcx_cmdq_in(cqe); 3624 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3625 in->cmd_op_mod = type; 3626 3627 mcx_cmdq_post(sc, cqe, 0); 3628 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3629 printf(", query pages timeout\n"); 3630 return (-1); 3631 } 3632 if (mcx_cmdq_verify(cqe) != 0) { 3633 printf(", query pages reply corrupt\n"); 3634 return (-1); 3635 } 3636 3637 out = mcx_cmdq_out(cqe); 3638 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3639 printf(", query pages failed (%x)\n", out->cmd_status); 3640 return (-1); 3641 } 3642 3643 *func_id = out->cmd_func_id; 3644 *npages = bemtoh32(&out->cmd_num_pages); 3645 3646 return (0); 3647 } 3648 3649 struct bus_dma_iter { 3650 bus_dmamap_t i_map; 3651 bus_size_t i_offset; 3652 unsigned int i_index; 3653 }; 3654 3655 static void 3656 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3657 { 3658 i->i_map = map; 3659 i->i_offset = 0; 3660 i->i_index = 0; 3661 } 3662 3663 static bus_addr_t 3664 bus_dma_iter_addr(struct bus_dma_iter *i) 3665 { 3666 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3667 } 3668 3669 static void 3670 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3671 { 3672 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3673 bus_size_t diff; 3674 3675 do { 3676 diff = seg->ds_len - i->i_offset; 3677 if (size < diff) 3678 break; 3679 3680 size -= diff; 3681 3682 seg++; 3683 3684 i->i_offset = 0; 3685 i->i_index++; 3686 } while (size > 0); 3687 3688 i->i_offset += size; 3689 } 3690 3691 static int 3692 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3693 { 3694 struct mcx_dmamem mxm; 3695 struct mcx_cmdq_entry *cqe; 3696 struct mcx_cmd_manage_pages_in *in; 3697 struct mcx_cmd_manage_pages_out *out; 3698 unsigned int paslen, nmb, i, j, npages; 3699 struct bus_dma_iter iter; 3700 uint64_t *pas; 3701 uint8_t status; 3702 uint8_t token = mcx_cmdq_token(sc); 3703 int error; 3704 3705 npages = mhm->mhm_npages; 3706 3707 paslen = sizeof(*pas) * npages; 3708 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3709 3710 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3711 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3712 3713 in = mcx_cmdq_in(cqe); 3714 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3715 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3716 in->cmd_func_id = func_id; 3717 htobem32(&in->cmd_input_num_entries, npages); 3718 3719 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3720 &cqe->cq_input_ptr, token) != 0) { 3721 printf(", unable to allocate manage pages mailboxen\n"); 3722 return (-1); 3723 } 3724 3725 bus_dma_iter_init(&iter, mhm->mhm_map); 3726 for (i = 0; i < nmb; i++) { 3727 unsigned int lim; 3728 3729 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3730 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3731 3732 for (j = 0; j < lim; j++) { 3733 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3734 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3735 } 3736 3737 npages -= lim; 3738 } 3739 3740 mcx_cmdq_mboxes_sign(&mxm, nmb); 3741 3742 mcx_cmdq_post(sc, cqe, 0); 3743 error = mcx_cmdq_poll(sc, cqe, 1000); 3744 if (error != 0) { 3745 printf(", manage pages timeout\n"); 3746 goto free; 3747 } 3748 error = mcx_cmdq_verify(cqe); 3749 if (error != 0) { 3750 printf(", manage pages reply corrupt\n"); 3751 goto free; 3752 } 3753 3754 status = cqe->cq_output_data[0]; 3755 if (status != MCX_CQ_STATUS_OK) { 3756 printf(", manage pages failed (%x)\n", status); 3757 error = -1; 3758 goto free; 3759 } 3760 3761 free: 3762 mcx_dmamem_free(sc, &mxm); 3763 3764 return (error); 3765 } 3766 3767 static int 3768 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3769 { 3770 int32_t npages; 3771 uint16_t func_id; 3772 3773 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3774 /* error printed by mcx_query_pages */ 3775 return (-1); 3776 } 3777 3778 if (npages < 1) 3779 return (0); 3780 3781 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3782 printf(", unable to allocate hwmem\n"); 3783 return (-1); 3784 } 3785 3786 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3787 printf(", unable to add hwmem\n"); 3788 goto free; 3789 } 3790 3791 return (0); 3792 3793 free: 3794 mcx_hwmem_free(sc, mhm); 3795 3796 return (-1); 3797 } 3798 3799 static int 3800 mcx_hca_max_caps(struct mcx_softc *sc) 3801 { 3802 struct mcx_dmamem mxm; 3803 struct mcx_cmdq_entry *cqe; 3804 struct mcx_cmd_query_hca_cap_in *in; 3805 struct mcx_cmd_query_hca_cap_out *out; 3806 struct mcx_cmdq_mailbox *mb; 3807 struct mcx_cap_device *hca; 3808 uint8_t status; 3809 uint8_t token = mcx_cmdq_token(sc); 3810 int error; 3811 3812 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3813 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3814 token); 3815 3816 in = mcx_cmdq_in(cqe); 3817 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3818 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3819 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3820 3821 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3822 &cqe->cq_output_ptr, token) != 0) { 3823 printf(", unable to allocate query hca caps mailboxen\n"); 3824 return (-1); 3825 } 3826 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3827 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3828 3829 mcx_cmdq_post(sc, cqe, 0); 3830 error = mcx_cmdq_poll(sc, cqe, 1000); 3831 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3832 3833 if (error != 0) { 3834 printf(", query hca caps timeout\n"); 3835 goto free; 3836 } 3837 error = mcx_cmdq_verify(cqe); 3838 if (error != 0) { 3839 printf(", query hca caps reply corrupt\n"); 3840 goto free; 3841 } 3842 3843 status = cqe->cq_output_data[0]; 3844 if (status != MCX_CQ_STATUS_OK) { 3845 printf(", query hca caps failed (%x)\n", status); 3846 error = -1; 3847 goto free; 3848 } 3849 3850 mb = mcx_cq_mbox(&mxm, 0); 3851 hca = mcx_cq_mbox_data(mb); 3852 3853 if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE) 3854 != MCX_CAP_DEVICE_PORT_TYPE_ETH) { 3855 printf(", not in ethernet mode\n"); 3856 error = -1; 3857 goto free; 3858 } 3859 if (hca->log_pg_sz > PAGE_SHIFT) { 3860 printf(", minimum system page shift %u is too large\n", 3861 hca->log_pg_sz); 3862 error = -1; 3863 goto free; 3864 } 3865 /* 3866 * blueflame register is split into two buffers, and we must alternate 3867 * between the two of them. 3868 */ 3869 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3870 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 3871 3872 sc->sc_mhz = bemtoh32(&hca->device_frequency_mhz); 3873 sc->sc_khz = bemtoh32(&hca->device_frequency_khz); 3874 3875 free: 3876 mcx_dmamem_free(sc, &mxm); 3877 3878 return (error); 3879 } 3880 3881 static int 3882 mcx_hca_set_caps(struct mcx_softc *sc) 3883 { 3884 struct mcx_dmamem mxm; 3885 struct mcx_cmdq_entry *cqe; 3886 struct mcx_cmd_query_hca_cap_in *in; 3887 struct mcx_cmd_query_hca_cap_out *out; 3888 struct mcx_cmdq_mailbox *mb; 3889 struct mcx_cap_device *hca; 3890 uint8_t status; 3891 uint8_t token = mcx_cmdq_token(sc); 3892 int error; 3893 3894 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3895 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3896 token); 3897 3898 in = mcx_cmdq_in(cqe); 3899 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3900 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 3901 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3902 3903 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3904 &cqe->cq_output_ptr, token) != 0) { 3905 printf(", unable to allocate manage pages mailboxen\n"); 3906 return (-1); 3907 } 3908 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3909 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3910 3911 mcx_cmdq_post(sc, cqe, 0); 3912 error = mcx_cmdq_poll(sc, cqe, 1000); 3913 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3914 3915 if (error != 0) { 3916 printf(", query hca caps timeout\n"); 3917 goto free; 3918 } 3919 error = mcx_cmdq_verify(cqe); 3920 if (error != 0) { 3921 printf(", query hca caps reply corrupt\n"); 3922 goto free; 3923 } 3924 3925 status = cqe->cq_output_data[0]; 3926 if (status != MCX_CQ_STATUS_OK) { 3927 printf(", query hca caps failed (%x)\n", status); 3928 error = -1; 3929 goto free; 3930 } 3931 3932 mb = mcx_cq_mbox(&mxm, 0); 3933 hca = mcx_cq_mbox_data(mb); 3934 3935 hca->log_pg_sz = PAGE_SHIFT; 3936 3937 free: 3938 mcx_dmamem_free(sc, &mxm); 3939 3940 return (error); 3941 } 3942 3943 3944 static int 3945 mcx_init_hca(struct mcx_softc *sc) 3946 { 3947 struct mcx_cmdq_entry *cqe; 3948 struct mcx_cmd_init_hca_in *in; 3949 struct mcx_cmd_init_hca_out *out; 3950 int error; 3951 uint8_t status; 3952 3953 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3954 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3955 3956 in = mcx_cmdq_in(cqe); 3957 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 3958 in->cmd_op_mod = htobe16(0); 3959 3960 mcx_cmdq_post(sc, cqe, 0); 3961 3962 error = mcx_cmdq_poll(sc, cqe, 1000); 3963 if (error != 0) { 3964 printf(", hca init timeout\n"); 3965 return (-1); 3966 } 3967 if (mcx_cmdq_verify(cqe) != 0) { 3968 printf(", hca init command corrupt\n"); 3969 return (-1); 3970 } 3971 3972 status = cqe->cq_output_data[0]; 3973 if (status != MCX_CQ_STATUS_OK) { 3974 printf(", hca init failed (%x)\n", status); 3975 return (-1); 3976 } 3977 3978 return (0); 3979 } 3980 3981 static int 3982 mcx_set_driver_version(struct mcx_softc *sc) 3983 { 3984 struct mcx_dmamem mxm; 3985 struct mcx_cmdq_entry *cqe; 3986 struct mcx_cmd_set_driver_version_in *in; 3987 struct mcx_cmd_set_driver_version_out *out; 3988 int error; 3989 int token; 3990 uint8_t status; 3991 3992 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3993 token = mcx_cmdq_token(sc); 3994 mcx_cmdq_init(sc, cqe, sizeof(*in) + 3995 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 3996 3997 in = mcx_cmdq_in(cqe); 3998 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 3999 in->cmd_op_mod = htobe16(0); 4000 4001 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4002 &cqe->cq_input_ptr, token) != 0) { 4003 printf(", unable to allocate set driver version mailboxen\n"); 4004 return (-1); 4005 } 4006 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 4007 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 4008 4009 mcx_cmdq_mboxes_sign(&mxm, 1); 4010 mcx_cmdq_post(sc, cqe, 0); 4011 4012 error = mcx_cmdq_poll(sc, cqe, 1000); 4013 if (error != 0) { 4014 printf(", set driver version timeout\n"); 4015 goto free; 4016 } 4017 if (mcx_cmdq_verify(cqe) != 0) { 4018 printf(", set driver version command corrupt\n"); 4019 goto free; 4020 } 4021 4022 status = cqe->cq_output_data[0]; 4023 if (status != MCX_CQ_STATUS_OK) { 4024 printf(", set driver version failed (%x)\n", status); 4025 error = -1; 4026 goto free; 4027 } 4028 4029 free: 4030 mcx_dmamem_free(sc, &mxm); 4031 4032 return (error); 4033 } 4034 4035 static int 4036 mcx_iff(struct mcx_softc *sc) 4037 { 4038 struct ifnet *ifp = &sc->sc_ac.ac_if; 4039 struct mcx_dmamem mxm; 4040 struct mcx_cmdq_entry *cqe; 4041 struct mcx_cmd_modify_nic_vport_context_in *in; 4042 struct mcx_cmd_modify_nic_vport_context_out *out; 4043 struct mcx_nic_vport_ctx *ctx; 4044 int error; 4045 int token; 4046 int insize; 4047 uint32_t dest; 4048 4049 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 4050 sc->sc_rss_flow_table_id; 4051 4052 /* enable or disable the promisc flow */ 4053 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 4054 if (sc->sc_promisc_flow_enabled == 0) { 4055 mcx_set_flow_table_entry_mac(sc, 4056 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 4057 sc->sc_promisc_flow_enabled = 1; 4058 } 4059 } else if (sc->sc_promisc_flow_enabled != 0) { 4060 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 4061 sc->sc_promisc_flow_enabled = 0; 4062 } 4063 4064 /* enable or disable the all-multicast flow */ 4065 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 4066 if (sc->sc_allmulti_flow_enabled == 0) { 4067 uint8_t mcast[ETHER_ADDR_LEN]; 4068 4069 memset(mcast, 0, sizeof(mcast)); 4070 mcast[0] = 0x01; 4071 mcx_set_flow_table_entry_mac(sc, 4072 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 4073 sc->sc_allmulti_flow_enabled = 1; 4074 } 4075 } else if (sc->sc_allmulti_flow_enabled != 0) { 4076 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 4077 sc->sc_allmulti_flow_enabled = 0; 4078 } 4079 4080 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 4081 4082 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4083 token = mcx_cmdq_token(sc); 4084 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4085 4086 in = mcx_cmdq_in(cqe); 4087 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 4088 in->cmd_op_mod = htobe16(0); 4089 in->cmd_field_select = htobe32( 4090 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 4091 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 4092 4093 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4094 printf(", unable to allocate modify " 4095 "nic vport context mailboxen\n"); 4096 return (-1); 4097 } 4098 ctx = (struct mcx_nic_vport_ctx *) 4099 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 4100 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 4101 /* 4102 * always leave promisc-all enabled on the vport since we 4103 * can't give it a vlan list, and we're already doing multicast 4104 * filtering in the flow table. 4105 */ 4106 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 4107 4108 mcx_cmdq_mboxes_sign(&mxm, 1); 4109 mcx_cmdq_post(sc, cqe, 0); 4110 4111 error = mcx_cmdq_poll(sc, cqe, 1000); 4112 if (error != 0) { 4113 printf(", modify nic vport context timeout\n"); 4114 goto free; 4115 } 4116 if (mcx_cmdq_verify(cqe) != 0) { 4117 printf(", modify nic vport context command corrupt\n"); 4118 goto free; 4119 } 4120 4121 out = mcx_cmdq_out(cqe); 4122 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4123 printf(", modify nic vport context failed (%x, %x)\n", 4124 out->cmd_status, betoh32(out->cmd_syndrome)); 4125 error = -1; 4126 goto free; 4127 } 4128 4129 free: 4130 mcx_dmamem_free(sc, &mxm); 4131 4132 return (error); 4133 } 4134 4135 static int 4136 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 4137 { 4138 struct mcx_cmdq_entry *cqe; 4139 struct mcx_cmd_alloc_uar_in *in; 4140 struct mcx_cmd_alloc_uar_out *out; 4141 int error; 4142 4143 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4144 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4145 4146 in = mcx_cmdq_in(cqe); 4147 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 4148 in->cmd_op_mod = htobe16(0); 4149 4150 mcx_cmdq_post(sc, cqe, 0); 4151 4152 error = mcx_cmdq_poll(sc, cqe, 1000); 4153 if (error != 0) { 4154 printf(", alloc uar timeout\n"); 4155 return (-1); 4156 } 4157 if (mcx_cmdq_verify(cqe) != 0) { 4158 printf(", alloc uar command corrupt\n"); 4159 return (-1); 4160 } 4161 4162 out = mcx_cmdq_out(cqe); 4163 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4164 printf(", alloc uar failed (%x)\n", out->cmd_status); 4165 return (-1); 4166 } 4167 4168 *uar = mcx_get_id(out->cmd_uar); 4169 return (0); 4170 } 4171 4172 static int 4173 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4174 uint64_t events, int vector) 4175 { 4176 struct mcx_cmdq_entry *cqe; 4177 struct mcx_dmamem mxm; 4178 struct mcx_cmd_create_eq_in *in; 4179 struct mcx_cmd_create_eq_mb_in *mbin; 4180 struct mcx_cmd_create_eq_out *out; 4181 struct mcx_eq_entry *eqe; 4182 int error; 4183 uint64_t *pas; 4184 int insize, npages, paslen, i, token; 4185 4186 eq->eq_cons = 0; 4187 4188 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4189 MCX_PAGE_SIZE); 4190 paslen = npages * sizeof(*pas); 4191 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4192 4193 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4194 MCX_PAGE_SIZE) != 0) { 4195 printf(", unable to allocate event queue memory\n"); 4196 return (-1); 4197 } 4198 4199 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4200 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4201 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4202 } 4203 4204 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4205 token = mcx_cmdq_token(sc); 4206 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4207 4208 in = mcx_cmdq_in(cqe); 4209 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4210 in->cmd_op_mod = htobe16(0); 4211 4212 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4213 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4214 &cqe->cq_input_ptr, token) != 0) { 4215 printf(", unable to allocate create eq mailboxen\n"); 4216 goto free_eq; 4217 } 4218 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4219 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4220 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4221 mbin->cmd_eq_ctx.eq_intr = vector; 4222 mbin->cmd_event_bitmask = htobe64(events); 4223 4224 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4225 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 4226 4227 /* physical addresses follow the mailbox in data */ 4228 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4229 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4230 mcx_cmdq_post(sc, cqe, 0); 4231 4232 error = mcx_cmdq_poll(sc, cqe, 1000); 4233 if (error != 0) { 4234 printf(", create eq timeout\n"); 4235 goto free_mxm; 4236 } 4237 if (mcx_cmdq_verify(cqe) != 0) { 4238 printf(", create eq command corrupt\n"); 4239 goto free_mxm; 4240 } 4241 4242 out = mcx_cmdq_out(cqe); 4243 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4244 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4245 betoh32(out->cmd_syndrome)); 4246 goto free_mxm; 4247 } 4248 4249 eq->eq_n = mcx_get_id(out->cmd_eqn); 4250 4251 mcx_dmamem_free(sc, &mxm); 4252 4253 mcx_arm_eq(sc, eq, uar); 4254 4255 return (0); 4256 4257 free_mxm: 4258 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4259 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 4260 mcx_dmamem_free(sc, &mxm); 4261 free_eq: 4262 mcx_dmamem_free(sc, &eq->eq_mem); 4263 return (-1); 4264 } 4265 4266 static int 4267 mcx_alloc_pd(struct mcx_softc *sc) 4268 { 4269 struct mcx_cmdq_entry *cqe; 4270 struct mcx_cmd_alloc_pd_in *in; 4271 struct mcx_cmd_alloc_pd_out *out; 4272 int error; 4273 4274 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4275 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4276 4277 in = mcx_cmdq_in(cqe); 4278 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4279 in->cmd_op_mod = htobe16(0); 4280 4281 mcx_cmdq_post(sc, cqe, 0); 4282 4283 error = mcx_cmdq_poll(sc, cqe, 1000); 4284 if (error != 0) { 4285 printf(", alloc pd timeout\n"); 4286 return (-1); 4287 } 4288 if (mcx_cmdq_verify(cqe) != 0) { 4289 printf(", alloc pd command corrupt\n"); 4290 return (-1); 4291 } 4292 4293 out = mcx_cmdq_out(cqe); 4294 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4295 printf(", alloc pd failed (%x)\n", out->cmd_status); 4296 return (-1); 4297 } 4298 4299 sc->sc_pd = mcx_get_id(out->cmd_pd); 4300 return (0); 4301 } 4302 4303 static int 4304 mcx_alloc_tdomain(struct mcx_softc *sc) 4305 { 4306 struct mcx_cmdq_entry *cqe; 4307 struct mcx_cmd_alloc_td_in *in; 4308 struct mcx_cmd_alloc_td_out *out; 4309 int error; 4310 4311 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4312 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4313 4314 in = mcx_cmdq_in(cqe); 4315 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4316 in->cmd_op_mod = htobe16(0); 4317 4318 mcx_cmdq_post(sc, cqe, 0); 4319 4320 error = mcx_cmdq_poll(sc, cqe, 1000); 4321 if (error != 0) { 4322 printf(", alloc transport domain timeout\n"); 4323 return (-1); 4324 } 4325 if (mcx_cmdq_verify(cqe) != 0) { 4326 printf(", alloc transport domain command corrupt\n"); 4327 return (-1); 4328 } 4329 4330 out = mcx_cmdq_out(cqe); 4331 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4332 printf(", alloc transport domain failed (%x)\n", 4333 out->cmd_status); 4334 return (-1); 4335 } 4336 4337 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4338 return (0); 4339 } 4340 4341 static int 4342 mcx_query_nic_vport_context(struct mcx_softc *sc) 4343 { 4344 struct mcx_dmamem mxm; 4345 struct mcx_cmdq_entry *cqe; 4346 struct mcx_cmd_query_nic_vport_context_in *in; 4347 struct mcx_cmd_query_nic_vport_context_out *out; 4348 struct mcx_nic_vport_ctx *ctx; 4349 uint8_t *addr; 4350 int error, token, i; 4351 4352 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4353 token = mcx_cmdq_token(sc); 4354 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4355 4356 in = mcx_cmdq_in(cqe); 4357 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4358 in->cmd_op_mod = htobe16(0); 4359 in->cmd_allowed_list_type = 0; 4360 4361 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4362 &cqe->cq_output_ptr, token) != 0) { 4363 printf(", unable to allocate " 4364 "query nic vport context mailboxen\n"); 4365 return (-1); 4366 } 4367 mcx_cmdq_mboxes_sign(&mxm, 1); 4368 mcx_cmdq_post(sc, cqe, 0); 4369 4370 error = mcx_cmdq_poll(sc, cqe, 1000); 4371 if (error != 0) { 4372 printf(", query nic vport context timeout\n"); 4373 goto free; 4374 } 4375 if (mcx_cmdq_verify(cqe) != 0) { 4376 printf(", query nic vport context command corrupt\n"); 4377 goto free; 4378 } 4379 4380 out = mcx_cmdq_out(cqe); 4381 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4382 printf(", query nic vport context failed (%x, %x)\n", 4383 out->cmd_status, betoh32(out->cmd_syndrome)); 4384 error = -1; 4385 goto free; 4386 } 4387 4388 ctx = (struct mcx_nic_vport_ctx *) 4389 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4390 addr = (uint8_t *)&ctx->vp_perm_addr; 4391 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4392 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 4393 } 4394 free: 4395 mcx_dmamem_free(sc, &mxm); 4396 4397 return (error); 4398 } 4399 4400 static int 4401 mcx_query_special_contexts(struct mcx_softc *sc) 4402 { 4403 struct mcx_cmdq_entry *cqe; 4404 struct mcx_cmd_query_special_ctx_in *in; 4405 struct mcx_cmd_query_special_ctx_out *out; 4406 int error; 4407 4408 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4409 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4410 4411 in = mcx_cmdq_in(cqe); 4412 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4413 in->cmd_op_mod = htobe16(0); 4414 4415 mcx_cmdq_post(sc, cqe, 0); 4416 4417 error = mcx_cmdq_poll(sc, cqe, 1000); 4418 if (error != 0) { 4419 printf(", query special contexts timeout\n"); 4420 return (-1); 4421 } 4422 if (mcx_cmdq_verify(cqe) != 0) { 4423 printf(", query special contexts command corrupt\n"); 4424 return (-1); 4425 } 4426 4427 out = mcx_cmdq_out(cqe); 4428 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4429 printf(", query special contexts failed (%x)\n", 4430 out->cmd_status); 4431 return (-1); 4432 } 4433 4434 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 4435 return (0); 4436 } 4437 4438 static int 4439 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4440 { 4441 struct mcx_reg_pmtu pmtu; 4442 int error; 4443 4444 /* read max mtu */ 4445 memset(&pmtu, 0, sizeof(pmtu)); 4446 pmtu.rp_local_port = 1; 4447 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4448 sizeof(pmtu)); 4449 if (error != 0) { 4450 printf(", unable to get port MTU\n"); 4451 return error; 4452 } 4453 4454 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 4455 pmtu.rp_admin_mtu = htobe16(mtu); 4456 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4457 sizeof(pmtu)); 4458 if (error != 0) { 4459 printf(", unable to set port MTU\n"); 4460 return error; 4461 } 4462 4463 sc->sc_hardmtu = mtu; 4464 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4465 return 0; 4466 } 4467 4468 static int 4469 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4470 { 4471 struct mcx_cmdq_entry *cmde; 4472 struct mcx_cq_entry *cqe; 4473 struct mcx_dmamem mxm; 4474 struct mcx_cmd_create_cq_in *in; 4475 struct mcx_cmd_create_cq_mb_in *mbin; 4476 struct mcx_cmd_create_cq_out *out; 4477 int error; 4478 uint64_t *pas; 4479 int insize, npages, paslen, i, token; 4480 4481 cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4482 4483 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4484 MCX_PAGE_SIZE); 4485 paslen = npages * sizeof(*pas); 4486 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4487 4488 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4489 MCX_PAGE_SIZE) != 0) { 4490 printf("%s: unable to allocate completion queue memory\n", 4491 DEVNAME(sc)); 4492 return (-1); 4493 } 4494 cqe = MCX_DMA_KVA(&cq->cq_mem); 4495 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4496 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4497 } 4498 4499 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4500 token = mcx_cmdq_token(sc); 4501 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4502 4503 in = mcx_cmdq_in(cmde); 4504 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4505 in->cmd_op_mod = htobe16(0); 4506 4507 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4508 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4509 &cmde->cq_input_ptr, token) != 0) { 4510 printf("%s: unable to allocate create cq mailboxen\n", 4511 DEVNAME(sc)); 4512 goto free_cq; 4513 } 4514 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4515 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4516 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4517 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4518 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4519 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4520 MCX_CQ_MOD_COUNTER); 4521 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4522 MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell); 4523 4524 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4525 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 4526 4527 /* physical addresses follow the mailbox in data */ 4528 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4529 mcx_cmdq_post(sc, cmde, 0); 4530 4531 error = mcx_cmdq_poll(sc, cmde, 1000); 4532 if (error != 0) { 4533 printf("%s: create cq timeout\n", DEVNAME(sc)); 4534 goto free_mxm; 4535 } 4536 if (mcx_cmdq_verify(cmde) != 0) { 4537 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4538 goto free_mxm; 4539 } 4540 4541 out = mcx_cmdq_out(cmde); 4542 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4543 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4544 out->cmd_status, betoh32(out->cmd_syndrome)); 4545 goto free_mxm; 4546 } 4547 4548 cq->cq_n = mcx_get_id(out->cmd_cqn); 4549 cq->cq_cons = 0; 4550 cq->cq_count = 0; 4551 4552 mcx_dmamem_free(sc, &mxm); 4553 4554 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4555 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4556 BUS_DMASYNC_PREWRITE); 4557 4558 mcx_arm_cq(sc, cq, uar); 4559 4560 return (0); 4561 4562 free_mxm: 4563 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4564 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4565 mcx_dmamem_free(sc, &mxm); 4566 free_cq: 4567 mcx_dmamem_free(sc, &cq->cq_mem); 4568 return (-1); 4569 } 4570 4571 static int 4572 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4573 { 4574 struct mcx_cmdq_entry *cqe; 4575 struct mcx_cmd_destroy_cq_in *in; 4576 struct mcx_cmd_destroy_cq_out *out; 4577 int error; 4578 int token; 4579 4580 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4581 token = mcx_cmdq_token(sc); 4582 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4583 4584 in = mcx_cmdq_in(cqe); 4585 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4586 in->cmd_op_mod = htobe16(0); 4587 in->cmd_cqn = htobe32(cq->cq_n); 4588 4589 mcx_cmdq_post(sc, cqe, 0); 4590 error = mcx_cmdq_poll(sc, cqe, 1000); 4591 if (error != 0) { 4592 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4593 return error; 4594 } 4595 if (mcx_cmdq_verify(cqe) != 0) { 4596 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4597 return error; 4598 } 4599 4600 out = mcx_cmdq_out(cqe); 4601 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4602 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4603 out->cmd_status, betoh32(out->cmd_syndrome)); 4604 return -1; 4605 } 4606 4607 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4608 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4609 BUS_DMASYNC_POSTWRITE); 4610 4611 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4612 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4613 mcx_dmamem_free(sc, &cq->cq_mem); 4614 4615 cq->cq_n = 0; 4616 cq->cq_cons = 0; 4617 cq->cq_count = 0; 4618 return 0; 4619 } 4620 4621 static int 4622 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4623 { 4624 struct mcx_cmdq_entry *cqe; 4625 struct mcx_dmamem mxm; 4626 struct mcx_cmd_create_rq_in *in; 4627 struct mcx_cmd_create_rq_out *out; 4628 struct mcx_rq_ctx *mbin; 4629 int error; 4630 uint64_t *pas; 4631 uint32_t rq_flags; 4632 int insize, npages, paslen, token; 4633 4634 rx->rx_doorbell = MCX_WQ_DOORBELL_BASE + 4635 (db * MCX_WQ_DOORBELL_STRIDE); 4636 4637 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4638 MCX_PAGE_SIZE); 4639 paslen = npages * sizeof(*pas); 4640 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4641 4642 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4643 MCX_PAGE_SIZE) != 0) { 4644 printf("%s: unable to allocate receive queue memory\n", 4645 DEVNAME(sc)); 4646 return (-1); 4647 } 4648 4649 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4650 token = mcx_cmdq_token(sc); 4651 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4652 4653 in = mcx_cmdq_in(cqe); 4654 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4655 in->cmd_op_mod = htobe16(0); 4656 4657 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4658 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4659 &cqe->cq_input_ptr, token) != 0) { 4660 printf("%s: unable to allocate create rq mailboxen\n", 4661 DEVNAME(sc)); 4662 goto free_rq; 4663 } 4664 mbin = (struct mcx_rq_ctx *) 4665 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4666 rq_flags = MCX_RQ_CTX_RLKEY; 4667 #if NVLAN == 0 4668 rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS; 4669 #endif 4670 mbin->rq_flags = htobe32(rq_flags); 4671 mbin->rq_cqn = htobe32(cqn); 4672 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4673 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4674 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4675 rx->rx_doorbell); 4676 mbin->rq_wq.wq_log_stride = htobe16(4); 4677 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4678 4679 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4680 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 4681 4682 /* physical addresses follow the mailbox in data */ 4683 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4684 mcx_cmdq_post(sc, cqe, 0); 4685 4686 error = mcx_cmdq_poll(sc, cqe, 1000); 4687 if (error != 0) { 4688 printf("%s: create rq timeout\n", DEVNAME(sc)); 4689 goto free_mxm; 4690 } 4691 if (mcx_cmdq_verify(cqe) != 0) { 4692 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4693 goto free_mxm; 4694 } 4695 4696 out = mcx_cmdq_out(cqe); 4697 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4698 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4699 out->cmd_status, betoh32(out->cmd_syndrome)); 4700 goto free_mxm; 4701 } 4702 4703 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4704 4705 mcx_dmamem_free(sc, &mxm); 4706 4707 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4708 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 4709 4710 return (0); 4711 4712 free_mxm: 4713 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4714 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4715 mcx_dmamem_free(sc, &mxm); 4716 free_rq: 4717 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4718 return (-1); 4719 } 4720 4721 static int 4722 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4723 { 4724 struct mcx_cmdq_entry *cqe; 4725 struct mcx_dmamem mxm; 4726 struct mcx_cmd_modify_rq_in *in; 4727 struct mcx_cmd_modify_rq_mb_in *mbin; 4728 struct mcx_cmd_modify_rq_out *out; 4729 int error; 4730 int token; 4731 4732 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4733 token = mcx_cmdq_token(sc); 4734 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4735 sizeof(*out), token); 4736 4737 in = mcx_cmdq_in(cqe); 4738 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4739 in->cmd_op_mod = htobe16(0); 4740 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4741 4742 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4743 &cqe->cq_input_ptr, token) != 0) { 4744 printf("%s: unable to allocate modify rq mailbox\n", 4745 DEVNAME(sc)); 4746 return (-1); 4747 } 4748 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4749 mbin->cmd_rq_ctx.rq_flags = htobe32( 4750 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4751 4752 mcx_cmdq_mboxes_sign(&mxm, 1); 4753 mcx_cmdq_post(sc, cqe, 0); 4754 error = mcx_cmdq_poll(sc, cqe, 1000); 4755 if (error != 0) { 4756 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4757 goto free; 4758 } 4759 if (mcx_cmdq_verify(cqe) != 0) { 4760 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4761 goto free; 4762 } 4763 4764 out = mcx_cmdq_out(cqe); 4765 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4766 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4767 out->cmd_status, betoh32(out->cmd_syndrome)); 4768 error = -1; 4769 goto free; 4770 } 4771 4772 free: 4773 mcx_dmamem_free(sc, &mxm); 4774 return (error); 4775 } 4776 4777 static int 4778 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4779 { 4780 struct mcx_cmdq_entry *cqe; 4781 struct mcx_cmd_destroy_rq_in *in; 4782 struct mcx_cmd_destroy_rq_out *out; 4783 int error; 4784 int token; 4785 4786 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4787 token = mcx_cmdq_token(sc); 4788 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4789 4790 in = mcx_cmdq_in(cqe); 4791 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4792 in->cmd_op_mod = htobe16(0); 4793 in->cmd_rqn = htobe32(rx->rx_rqn); 4794 4795 mcx_cmdq_post(sc, cqe, 0); 4796 error = mcx_cmdq_poll(sc, cqe, 1000); 4797 if (error != 0) { 4798 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4799 return error; 4800 } 4801 if (mcx_cmdq_verify(cqe) != 0) { 4802 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4803 return error; 4804 } 4805 4806 out = mcx_cmdq_out(cqe); 4807 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4808 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4809 out->cmd_status, betoh32(out->cmd_syndrome)); 4810 return -1; 4811 } 4812 4813 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4814 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 4815 4816 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4817 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4818 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4819 4820 rx->rx_rqn = 0; 4821 return 0; 4822 } 4823 4824 static int 4825 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 4826 { 4827 struct mcx_cmdq_entry *cqe; 4828 struct mcx_dmamem mxm; 4829 struct mcx_cmd_create_tir_in *in; 4830 struct mcx_cmd_create_tir_mb_in *mbin; 4831 struct mcx_cmd_create_tir_out *out; 4832 int error; 4833 int token; 4834 4835 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4836 token = mcx_cmdq_token(sc); 4837 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4838 sizeof(*out), token); 4839 4840 in = mcx_cmdq_in(cqe); 4841 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4842 in->cmd_op_mod = htobe16(0); 4843 4844 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4845 &cqe->cq_input_ptr, token) != 0) { 4846 printf("%s: unable to allocate create tir mailbox\n", 4847 DEVNAME(sc)); 4848 return (-1); 4849 } 4850 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4851 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4852 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 4853 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4854 4855 mcx_cmdq_post(sc, cqe, 0); 4856 error = mcx_cmdq_poll(sc, cqe, 1000); 4857 if (error != 0) { 4858 printf("%s: create tir timeout\n", DEVNAME(sc)); 4859 goto free; 4860 } 4861 if (mcx_cmdq_verify(cqe) != 0) { 4862 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4863 goto free; 4864 } 4865 4866 out = mcx_cmdq_out(cqe); 4867 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4868 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4869 out->cmd_status, betoh32(out->cmd_syndrome)); 4870 error = -1; 4871 goto free; 4872 } 4873 4874 *tirn = mcx_get_id(out->cmd_tirn); 4875 free: 4876 mcx_dmamem_free(sc, &mxm); 4877 return (error); 4878 } 4879 4880 static int 4881 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 4882 int *tirn) 4883 { 4884 struct mcx_cmdq_entry *cqe; 4885 struct mcx_dmamem mxm; 4886 struct mcx_cmd_create_tir_in *in; 4887 struct mcx_cmd_create_tir_mb_in *mbin; 4888 struct mcx_cmd_create_tir_out *out; 4889 int error; 4890 int token; 4891 4892 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4893 token = mcx_cmdq_token(sc); 4894 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4895 sizeof(*out), token); 4896 4897 in = mcx_cmdq_in(cqe); 4898 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4899 in->cmd_op_mod = htobe16(0); 4900 4901 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4902 &cqe->cq_input_ptr, token) != 0) { 4903 printf("%s: unable to allocate create tir mailbox\n", 4904 DEVNAME(sc)); 4905 return (-1); 4906 } 4907 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4908 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 4909 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 4910 mbin->cmd_indir_table = htobe32(rqtn); 4911 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 4912 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 4913 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 4914 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 4915 sizeof(mbin->cmd_rx_hash_key)); 4916 4917 mcx_cmdq_post(sc, cqe, 0); 4918 error = mcx_cmdq_poll(sc, cqe, 1000); 4919 if (error != 0) { 4920 printf("%s: create tir timeout\n", DEVNAME(sc)); 4921 goto free; 4922 } 4923 if (mcx_cmdq_verify(cqe) != 0) { 4924 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4925 goto free; 4926 } 4927 4928 out = mcx_cmdq_out(cqe); 4929 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4930 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4931 out->cmd_status, betoh32(out->cmd_syndrome)); 4932 error = -1; 4933 goto free; 4934 } 4935 4936 *tirn = mcx_get_id(out->cmd_tirn); 4937 free: 4938 mcx_dmamem_free(sc, &mxm); 4939 return (error); 4940 } 4941 4942 static int 4943 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 4944 { 4945 struct mcx_cmdq_entry *cqe; 4946 struct mcx_cmd_destroy_tir_in *in; 4947 struct mcx_cmd_destroy_tir_out *out; 4948 int error; 4949 int token; 4950 4951 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4952 token = mcx_cmdq_token(sc); 4953 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4954 4955 in = mcx_cmdq_in(cqe); 4956 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 4957 in->cmd_op_mod = htobe16(0); 4958 in->cmd_tirn = htobe32(tirn); 4959 4960 mcx_cmdq_post(sc, cqe, 0); 4961 error = mcx_cmdq_poll(sc, cqe, 1000); 4962 if (error != 0) { 4963 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 4964 return error; 4965 } 4966 if (mcx_cmdq_verify(cqe) != 0) { 4967 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 4968 return error; 4969 } 4970 4971 out = mcx_cmdq_out(cqe); 4972 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4973 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 4974 out->cmd_status, betoh32(out->cmd_syndrome)); 4975 return -1; 4976 } 4977 4978 return (0); 4979 } 4980 4981 static int 4982 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 4983 int cqn) 4984 { 4985 struct mcx_cmdq_entry *cqe; 4986 struct mcx_dmamem mxm; 4987 struct mcx_cmd_create_sq_in *in; 4988 struct mcx_sq_ctx *mbin; 4989 struct mcx_cmd_create_sq_out *out; 4990 int error; 4991 uint64_t *pas; 4992 int insize, npages, paslen, token; 4993 4994 tx->tx_doorbell = MCX_WQ_DOORBELL_BASE + 4995 (db * MCX_WQ_DOORBELL_STRIDE) + 4; 4996 4997 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 4998 MCX_PAGE_SIZE); 4999 paslen = npages * sizeof(*pas); 5000 insize = sizeof(struct mcx_sq_ctx) + paslen; 5001 5002 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 5003 MCX_PAGE_SIZE) != 0) { 5004 printf("%s: unable to allocate send queue memory\n", 5005 DEVNAME(sc)); 5006 return (-1); 5007 } 5008 5009 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5010 token = mcx_cmdq_token(sc); 5011 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 5012 token); 5013 5014 in = mcx_cmdq_in(cqe); 5015 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 5016 in->cmd_op_mod = htobe16(0); 5017 5018 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 5019 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 5020 &cqe->cq_input_ptr, token) != 0) { 5021 printf("%s: unable to allocate create sq mailboxen\n", 5022 DEVNAME(sc)); 5023 goto free_sq; 5024 } 5025 mbin = (struct mcx_sq_ctx *) 5026 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 5027 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 5028 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 5029 mbin->sq_cqn = htobe32(cqn); 5030 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 5031 mbin->sq_tis_num = htobe32(sc->sc_tis); 5032 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 5033 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 5034 mbin->sq_wq.wq_uar_page = htobe32(uar); 5035 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 5036 tx->tx_doorbell); 5037 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 5038 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 5039 5040 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5041 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 5042 5043 /* physical addresses follow the mailbox in data */ 5044 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 5045 npages, &tx->tx_sq_mem); 5046 mcx_cmdq_post(sc, cqe, 0); 5047 5048 error = mcx_cmdq_poll(sc, cqe, 1000); 5049 if (error != 0) { 5050 printf("%s: create sq timeout\n", DEVNAME(sc)); 5051 goto free_mxm; 5052 } 5053 if (mcx_cmdq_verify(cqe) != 0) { 5054 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 5055 goto free_mxm; 5056 } 5057 5058 out = mcx_cmdq_out(cqe); 5059 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5060 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 5061 out->cmd_status, betoh32(out->cmd_syndrome)); 5062 goto free_mxm; 5063 } 5064 5065 tx->tx_uar = uar; 5066 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 5067 5068 mcx_dmamem_free(sc, &mxm); 5069 5070 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5071 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 5072 5073 return (0); 5074 5075 free_mxm: 5076 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5077 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5078 mcx_dmamem_free(sc, &mxm); 5079 free_sq: 5080 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5081 return (-1); 5082 } 5083 5084 static int 5085 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5086 { 5087 struct mcx_cmdq_entry *cqe; 5088 struct mcx_cmd_destroy_sq_in *in; 5089 struct mcx_cmd_destroy_sq_out *out; 5090 int error; 5091 int token; 5092 5093 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5094 token = mcx_cmdq_token(sc); 5095 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5096 5097 in = mcx_cmdq_in(cqe); 5098 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 5099 in->cmd_op_mod = htobe16(0); 5100 in->cmd_sqn = htobe32(tx->tx_sqn); 5101 5102 mcx_cmdq_post(sc, cqe, 0); 5103 error = mcx_cmdq_poll(sc, cqe, 1000); 5104 if (error != 0) { 5105 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 5106 return error; 5107 } 5108 if (mcx_cmdq_verify(cqe) != 0) { 5109 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 5110 return error; 5111 } 5112 5113 out = mcx_cmdq_out(cqe); 5114 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5115 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 5116 out->cmd_status, betoh32(out->cmd_syndrome)); 5117 return -1; 5118 } 5119 5120 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5121 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 5122 5123 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5124 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5125 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5126 5127 tx->tx_sqn = 0; 5128 return 0; 5129 } 5130 5131 static int 5132 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5133 { 5134 struct mcx_cmdq_entry *cqe; 5135 struct mcx_dmamem mxm; 5136 struct mcx_cmd_modify_sq_in *in; 5137 struct mcx_cmd_modify_sq_mb_in *mbin; 5138 struct mcx_cmd_modify_sq_out *out; 5139 int error; 5140 int token; 5141 5142 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5143 token = mcx_cmdq_token(sc); 5144 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5145 sizeof(*out), token); 5146 5147 in = mcx_cmdq_in(cqe); 5148 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 5149 in->cmd_op_mod = htobe16(0); 5150 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 5151 5152 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5153 &cqe->cq_input_ptr, token) != 0) { 5154 printf("%s: unable to allocate modify sq mailbox\n", 5155 DEVNAME(sc)); 5156 return (-1); 5157 } 5158 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5159 mbin->cmd_sq_ctx.sq_flags = htobe32( 5160 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 5161 5162 mcx_cmdq_mboxes_sign(&mxm, 1); 5163 mcx_cmdq_post(sc, cqe, 0); 5164 error = mcx_cmdq_poll(sc, cqe, 1000); 5165 if (error != 0) { 5166 printf("%s: modify sq timeout\n", DEVNAME(sc)); 5167 goto free; 5168 } 5169 if (mcx_cmdq_verify(cqe) != 0) { 5170 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 5171 goto free; 5172 } 5173 5174 out = mcx_cmdq_out(cqe); 5175 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5176 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 5177 out->cmd_status, betoh32(out->cmd_syndrome)); 5178 error = -1; 5179 goto free; 5180 } 5181 5182 free: 5183 mcx_dmamem_free(sc, &mxm); 5184 return (error); 5185 } 5186 5187 static int 5188 mcx_create_tis(struct mcx_softc *sc, int *tis) 5189 { 5190 struct mcx_cmdq_entry *cqe; 5191 struct mcx_dmamem mxm; 5192 struct mcx_cmd_create_tis_in *in; 5193 struct mcx_cmd_create_tis_mb_in *mbin; 5194 struct mcx_cmd_create_tis_out *out; 5195 int error; 5196 int token; 5197 5198 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5199 token = mcx_cmdq_token(sc); 5200 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5201 sizeof(*out), token); 5202 5203 in = mcx_cmdq_in(cqe); 5204 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 5205 in->cmd_op_mod = htobe16(0); 5206 5207 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5208 &cqe->cq_input_ptr, token) != 0) { 5209 printf("%s: unable to allocate create tis mailbox\n", 5210 DEVNAME(sc)); 5211 return (-1); 5212 } 5213 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5214 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5215 5216 mcx_cmdq_mboxes_sign(&mxm, 1); 5217 mcx_cmdq_post(sc, cqe, 0); 5218 error = mcx_cmdq_poll(sc, cqe, 1000); 5219 if (error != 0) { 5220 printf("%s: create tis timeout\n", DEVNAME(sc)); 5221 goto free; 5222 } 5223 if (mcx_cmdq_verify(cqe) != 0) { 5224 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5225 goto free; 5226 } 5227 5228 out = mcx_cmdq_out(cqe); 5229 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5230 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5231 out->cmd_status, betoh32(out->cmd_syndrome)); 5232 error = -1; 5233 goto free; 5234 } 5235 5236 *tis = mcx_get_id(out->cmd_tisn); 5237 free: 5238 mcx_dmamem_free(sc, &mxm); 5239 return (error); 5240 } 5241 5242 static int 5243 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5244 { 5245 struct mcx_cmdq_entry *cqe; 5246 struct mcx_cmd_destroy_tis_in *in; 5247 struct mcx_cmd_destroy_tis_out *out; 5248 int error; 5249 int token; 5250 5251 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5252 token = mcx_cmdq_token(sc); 5253 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5254 5255 in = mcx_cmdq_in(cqe); 5256 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5257 in->cmd_op_mod = htobe16(0); 5258 in->cmd_tisn = htobe32(tis); 5259 5260 mcx_cmdq_post(sc, cqe, 0); 5261 error = mcx_cmdq_poll(sc, cqe, 1000); 5262 if (error != 0) { 5263 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5264 return error; 5265 } 5266 if (mcx_cmdq_verify(cqe) != 0) { 5267 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5268 return error; 5269 } 5270 5271 out = mcx_cmdq_out(cqe); 5272 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5273 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5274 out->cmd_status, betoh32(out->cmd_syndrome)); 5275 return -1; 5276 } 5277 5278 return 0; 5279 } 5280 5281 static int 5282 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5283 { 5284 struct mcx_cmdq_entry *cqe; 5285 struct mcx_dmamem mxm; 5286 struct mcx_cmd_create_rqt_in *in; 5287 struct mcx_cmd_create_rqt_mb_in *mbin; 5288 struct mcx_cmd_create_rqt_out *out; 5289 struct mcx_rqt_ctx *rqt_ctx; 5290 int *rqtn; 5291 int error; 5292 int token; 5293 int i; 5294 5295 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5296 token = mcx_cmdq_token(sc); 5297 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5298 (size * sizeof(int)), sizeof(*out), token); 5299 5300 in = mcx_cmdq_in(cqe); 5301 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5302 in->cmd_op_mod = htobe16(0); 5303 5304 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5305 &cqe->cq_input_ptr, token) != 0) { 5306 printf("%s: unable to allocate create rqt mailbox\n", 5307 DEVNAME(sc)); 5308 return (-1); 5309 } 5310 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5311 rqt_ctx = &mbin->cmd_rqt; 5312 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5313 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5314 5315 /* rqt list follows the rqt context */ 5316 rqtn = (int *)(rqt_ctx + 1); 5317 for (i = 0; i < size; i++) { 5318 rqtn[i] = htobe32(rqns[i]); 5319 } 5320 5321 mcx_cmdq_mboxes_sign(&mxm, 1); 5322 mcx_cmdq_post(sc, cqe, 0); 5323 error = mcx_cmdq_poll(sc, cqe, 1000); 5324 if (error != 0) { 5325 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5326 goto free; 5327 } 5328 if (mcx_cmdq_verify(cqe) != 0) { 5329 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5330 goto free; 5331 } 5332 5333 out = mcx_cmdq_out(cqe); 5334 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5335 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5336 out->cmd_status, betoh32(out->cmd_syndrome)); 5337 error = -1; 5338 goto free; 5339 } 5340 5341 *rqt = mcx_get_id(out->cmd_rqtn); 5342 return (0); 5343 free: 5344 mcx_dmamem_free(sc, &mxm); 5345 return (error); 5346 } 5347 5348 static int 5349 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5350 { 5351 struct mcx_cmdq_entry *cqe; 5352 struct mcx_cmd_destroy_rqt_in *in; 5353 struct mcx_cmd_destroy_rqt_out *out; 5354 int error; 5355 int token; 5356 5357 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5358 token = mcx_cmdq_token(sc); 5359 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5360 5361 in = mcx_cmdq_in(cqe); 5362 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5363 in->cmd_op_mod = htobe16(0); 5364 in->cmd_rqtn = htobe32(rqt); 5365 5366 mcx_cmdq_post(sc, cqe, 0); 5367 error = mcx_cmdq_poll(sc, cqe, 1000); 5368 if (error != 0) { 5369 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5370 return error; 5371 } 5372 if (mcx_cmdq_verify(cqe) != 0) { 5373 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5374 return error; 5375 } 5376 5377 out = mcx_cmdq_out(cqe); 5378 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5379 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5380 out->cmd_status, betoh32(out->cmd_syndrome)); 5381 return -1; 5382 } 5383 5384 return 0; 5385 } 5386 5387 #if 0 5388 static int 5389 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5390 { 5391 struct mcx_cmdq_entry *cqe; 5392 struct mcx_cmd_alloc_flow_counter_in *in; 5393 struct mcx_cmd_alloc_flow_counter_out *out; 5394 int error; 5395 5396 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5397 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5398 5399 in = mcx_cmdq_in(cqe); 5400 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5401 in->cmd_op_mod = htobe16(0); 5402 5403 mcx_cmdq_post(sc, cqe, 0); 5404 5405 error = mcx_cmdq_poll(sc, cqe, 1000); 5406 if (error != 0) { 5407 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5408 return (-1); 5409 } 5410 if (mcx_cmdq_verify(cqe) != 0) { 5411 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5412 return (-1); 5413 } 5414 5415 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5416 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5417 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5418 out->cmd_status); 5419 return (-1); 5420 } 5421 5422 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 5423 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5424 5425 return (0); 5426 } 5427 #endif 5428 5429 static int 5430 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5431 int *flow_table_id) 5432 { 5433 struct mcx_cmdq_entry *cqe; 5434 struct mcx_dmamem mxm; 5435 struct mcx_cmd_create_flow_table_in *in; 5436 struct mcx_cmd_create_flow_table_mb_in *mbin; 5437 struct mcx_cmd_create_flow_table_out *out; 5438 int error; 5439 int token; 5440 5441 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5442 token = mcx_cmdq_token(sc); 5443 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5444 sizeof(*out), token); 5445 5446 in = mcx_cmdq_in(cqe); 5447 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5448 in->cmd_op_mod = htobe16(0); 5449 5450 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5451 &cqe->cq_input_ptr, token) != 0) { 5452 printf("%s: unable to allocate create flow table mailbox\n", 5453 DEVNAME(sc)); 5454 return (-1); 5455 } 5456 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5457 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5458 mbin->cmd_ctx.ft_log_size = log_size; 5459 mbin->cmd_ctx.ft_level = level; 5460 5461 mcx_cmdq_mboxes_sign(&mxm, 1); 5462 mcx_cmdq_post(sc, cqe, 0); 5463 error = mcx_cmdq_poll(sc, cqe, 1000); 5464 if (error != 0) { 5465 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5466 goto free; 5467 } 5468 if (mcx_cmdq_verify(cqe) != 0) { 5469 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5470 goto free; 5471 } 5472 5473 out = mcx_cmdq_out(cqe); 5474 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5475 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5476 out->cmd_status, betoh32(out->cmd_syndrome)); 5477 error = -1; 5478 goto free; 5479 } 5480 5481 *flow_table_id = mcx_get_id(out->cmd_table_id); 5482 free: 5483 mcx_dmamem_free(sc, &mxm); 5484 return (error); 5485 } 5486 5487 static int 5488 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5489 { 5490 struct mcx_cmdq_entry *cqe; 5491 struct mcx_dmamem mxm; 5492 struct mcx_cmd_set_flow_table_root_in *in; 5493 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5494 struct mcx_cmd_set_flow_table_root_out *out; 5495 int error; 5496 int token; 5497 5498 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5499 token = mcx_cmdq_token(sc); 5500 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5501 sizeof(*out), token); 5502 5503 in = mcx_cmdq_in(cqe); 5504 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5505 in->cmd_op_mod = htobe16(0); 5506 5507 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5508 &cqe->cq_input_ptr, token) != 0) { 5509 printf("%s: unable to allocate set flow table root mailbox\n", 5510 DEVNAME(sc)); 5511 return (-1); 5512 } 5513 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5514 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5515 mbin->cmd_table_id = htobe32(flow_table_id); 5516 5517 mcx_cmdq_mboxes_sign(&mxm, 1); 5518 mcx_cmdq_post(sc, cqe, 0); 5519 error = mcx_cmdq_poll(sc, cqe, 1000); 5520 if (error != 0) { 5521 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5522 goto free; 5523 } 5524 if (mcx_cmdq_verify(cqe) != 0) { 5525 printf("%s: set flow table root command corrupt\n", 5526 DEVNAME(sc)); 5527 goto free; 5528 } 5529 5530 out = mcx_cmdq_out(cqe); 5531 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5532 printf("%s: set flow table root failed (%x, %x)\n", 5533 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5534 error = -1; 5535 goto free; 5536 } 5537 5538 free: 5539 mcx_dmamem_free(sc, &mxm); 5540 return (error); 5541 } 5542 5543 static int 5544 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5545 { 5546 struct mcx_cmdq_entry *cqe; 5547 struct mcx_dmamem mxm; 5548 struct mcx_cmd_destroy_flow_table_in *in; 5549 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5550 struct mcx_cmd_destroy_flow_table_out *out; 5551 int error; 5552 int token; 5553 5554 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5555 token = mcx_cmdq_token(sc); 5556 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5557 5558 in = mcx_cmdq_in(cqe); 5559 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5560 in->cmd_op_mod = htobe16(0); 5561 5562 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5563 &cqe->cq_input_ptr, token) != 0) { 5564 printf("%s: unable to allocate destroy flow table mailbox\n", 5565 DEVNAME(sc)); 5566 return (-1); 5567 } 5568 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5569 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5570 mb->cmd_table_id = htobe32(flow_table_id); 5571 5572 mcx_cmdq_mboxes_sign(&mxm, 1); 5573 mcx_cmdq_post(sc, cqe, 0); 5574 error = mcx_cmdq_poll(sc, cqe, 1000); 5575 if (error != 0) { 5576 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5577 goto free; 5578 } 5579 if (mcx_cmdq_verify(cqe) != 0) { 5580 printf("%s: destroy flow table command corrupt\n", 5581 DEVNAME(sc)); 5582 goto free; 5583 } 5584 5585 out = mcx_cmdq_out(cqe); 5586 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5587 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5588 out->cmd_status, betoh32(out->cmd_syndrome)); 5589 error = -1; 5590 goto free; 5591 } 5592 5593 free: 5594 mcx_dmamem_free(sc, &mxm); 5595 return (error); 5596 } 5597 5598 5599 static int 5600 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5601 int start, int size, int match_enable, struct mcx_flow_match *match) 5602 { 5603 struct mcx_cmdq_entry *cqe; 5604 struct mcx_dmamem mxm; 5605 struct mcx_cmd_create_flow_group_in *in; 5606 struct mcx_cmd_create_flow_group_mb_in *mbin; 5607 struct mcx_cmd_create_flow_group_out *out; 5608 struct mcx_flow_group *mfg; 5609 int error; 5610 int token; 5611 5612 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5613 token = mcx_cmdq_token(sc); 5614 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5615 token); 5616 5617 in = mcx_cmdq_in(cqe); 5618 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5619 in->cmd_op_mod = htobe16(0); 5620 5621 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5622 != 0) { 5623 printf("%s: unable to allocate create flow group mailbox\n", 5624 DEVNAME(sc)); 5625 return (-1); 5626 } 5627 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5628 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5629 mbin->cmd_table_id = htobe32(flow_table_id); 5630 mbin->cmd_start_flow_index = htobe32(start); 5631 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5632 5633 mbin->cmd_match_criteria_enable = match_enable; 5634 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5635 5636 mcx_cmdq_mboxes_sign(&mxm, 2); 5637 mcx_cmdq_post(sc, cqe, 0); 5638 error = mcx_cmdq_poll(sc, cqe, 1000); 5639 if (error != 0) { 5640 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5641 goto free; 5642 } 5643 if (mcx_cmdq_verify(cqe) != 0) { 5644 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5645 goto free; 5646 } 5647 5648 out = mcx_cmdq_out(cqe); 5649 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5650 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5651 out->cmd_status, betoh32(out->cmd_syndrome)); 5652 error = -1; 5653 goto free; 5654 } 5655 5656 mfg = &sc->sc_flow_group[group]; 5657 mfg->g_id = mcx_get_id(out->cmd_group_id); 5658 mfg->g_table = flow_table_id; 5659 mfg->g_start = start; 5660 mfg->g_size = size; 5661 5662 free: 5663 mcx_dmamem_free(sc, &mxm); 5664 return (error); 5665 } 5666 5667 static int 5668 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5669 { 5670 struct mcx_cmdq_entry *cqe; 5671 struct mcx_dmamem mxm; 5672 struct mcx_cmd_destroy_flow_group_in *in; 5673 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5674 struct mcx_cmd_destroy_flow_group_out *out; 5675 struct mcx_flow_group *mfg; 5676 int error; 5677 int token; 5678 5679 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5680 token = mcx_cmdq_token(sc); 5681 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5682 5683 in = mcx_cmdq_in(cqe); 5684 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5685 in->cmd_op_mod = htobe16(0); 5686 5687 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5688 &cqe->cq_input_ptr, token) != 0) { 5689 printf("%s: unable to allocate destroy flow group mailbox\n", 5690 DEVNAME(sc)); 5691 return (-1); 5692 } 5693 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5694 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5695 mfg = &sc->sc_flow_group[group]; 5696 mb->cmd_table_id = htobe32(mfg->g_table); 5697 mb->cmd_group_id = htobe32(mfg->g_id); 5698 5699 mcx_cmdq_mboxes_sign(&mxm, 2); 5700 mcx_cmdq_post(sc, cqe, 0); 5701 error = mcx_cmdq_poll(sc, cqe, 1000); 5702 if (error != 0) { 5703 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5704 goto free; 5705 } 5706 if (mcx_cmdq_verify(cqe) != 0) { 5707 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5708 goto free; 5709 } 5710 5711 out = mcx_cmdq_out(cqe); 5712 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5713 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5714 out->cmd_status, betoh32(out->cmd_syndrome)); 5715 error = -1; 5716 goto free; 5717 } 5718 5719 mfg->g_id = -1; 5720 mfg->g_table = -1; 5721 mfg->g_size = 0; 5722 mfg->g_start = 0; 5723 free: 5724 mcx_dmamem_free(sc, &mxm); 5725 return (error); 5726 } 5727 5728 static int 5729 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5730 uint8_t *macaddr, uint32_t dest) 5731 { 5732 struct mcx_cmdq_entry *cqe; 5733 struct mcx_dmamem mxm; 5734 struct mcx_cmd_set_flow_table_entry_in *in; 5735 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5736 struct mcx_cmd_set_flow_table_entry_out *out; 5737 struct mcx_flow_group *mfg; 5738 uint32_t *pdest; 5739 int error; 5740 int token; 5741 5742 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5743 token = mcx_cmdq_token(sc); 5744 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5745 sizeof(*out), token); 5746 5747 in = mcx_cmdq_in(cqe); 5748 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5749 in->cmd_op_mod = htobe16(0); 5750 5751 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5752 != 0) { 5753 printf("%s: unable to allocate set flow table entry mailbox\n", 5754 DEVNAME(sc)); 5755 return (-1); 5756 } 5757 5758 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5759 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5760 5761 mfg = &sc->sc_flow_group[group]; 5762 mbin->cmd_table_id = htobe32(mfg->g_table); 5763 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5764 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5765 5766 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5767 pdest = (uint32_t *) 5768 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5769 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5770 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5771 *pdest = htobe32(dest); 5772 5773 /* the only thing we match on at the moment is the dest mac address */ 5774 if (macaddr != NULL) { 5775 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5776 ETHER_ADDR_LEN); 5777 } 5778 5779 mcx_cmdq_mboxes_sign(&mxm, 2); 5780 mcx_cmdq_post(sc, cqe, 0); 5781 error = mcx_cmdq_poll(sc, cqe, 1000); 5782 if (error != 0) { 5783 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5784 goto free; 5785 } 5786 if (mcx_cmdq_verify(cqe) != 0) { 5787 printf("%s: set flow table entry command corrupt\n", 5788 DEVNAME(sc)); 5789 goto free; 5790 } 5791 5792 out = mcx_cmdq_out(cqe); 5793 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5794 printf("%s: set flow table entry failed (%x, %x)\n", 5795 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5796 error = -1; 5797 goto free; 5798 } 5799 5800 free: 5801 mcx_dmamem_free(sc, &mxm); 5802 return (error); 5803 } 5804 5805 static int 5806 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5807 int ethertype, int ip_proto, uint32_t dest) 5808 { 5809 struct mcx_cmdq_entry *cqe; 5810 struct mcx_dmamem mxm; 5811 struct mcx_cmd_set_flow_table_entry_in *in; 5812 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5813 struct mcx_cmd_set_flow_table_entry_out *out; 5814 struct mcx_flow_group *mfg; 5815 uint32_t *pdest; 5816 int error; 5817 int token; 5818 5819 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5820 token = mcx_cmdq_token(sc); 5821 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5822 sizeof(*out), token); 5823 5824 in = mcx_cmdq_in(cqe); 5825 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5826 in->cmd_op_mod = htobe16(0); 5827 5828 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5829 != 0) { 5830 printf("%s: unable to allocate set flow table entry mailbox\n", 5831 DEVNAME(sc)); 5832 return (-1); 5833 } 5834 5835 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5836 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5837 5838 mfg = &sc->sc_flow_group[group]; 5839 mbin->cmd_table_id = htobe32(mfg->g_table); 5840 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5841 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5842 5843 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5844 pdest = (uint32_t *) 5845 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5846 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5847 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5848 *pdest = htobe32(dest); 5849 5850 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 5851 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 5852 5853 mcx_cmdq_mboxes_sign(&mxm, 2); 5854 mcx_cmdq_post(sc, cqe, 0); 5855 error = mcx_cmdq_poll(sc, cqe, 1000); 5856 if (error != 0) { 5857 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5858 goto free; 5859 } 5860 if (mcx_cmdq_verify(cqe) != 0) { 5861 printf("%s: set flow table entry command corrupt\n", 5862 DEVNAME(sc)); 5863 goto free; 5864 } 5865 5866 out = mcx_cmdq_out(cqe); 5867 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5868 printf("%s: set flow table entry failed (%x, %x)\n", 5869 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5870 error = -1; 5871 goto free; 5872 } 5873 5874 free: 5875 mcx_dmamem_free(sc, &mxm); 5876 return (error); 5877 } 5878 5879 static int 5880 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 5881 { 5882 struct mcx_cmdq_entry *cqe; 5883 struct mcx_dmamem mxm; 5884 struct mcx_cmd_delete_flow_table_entry_in *in; 5885 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 5886 struct mcx_cmd_delete_flow_table_entry_out *out; 5887 struct mcx_flow_group *mfg; 5888 int error; 5889 int token; 5890 5891 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5892 token = mcx_cmdq_token(sc); 5893 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5894 token); 5895 5896 in = mcx_cmdq_in(cqe); 5897 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 5898 in->cmd_op_mod = htobe16(0); 5899 5900 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5901 &cqe->cq_input_ptr, token) != 0) { 5902 printf("%s: unable to allocate " 5903 "delete flow table entry mailbox\n", DEVNAME(sc)); 5904 return (-1); 5905 } 5906 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5907 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5908 5909 mfg = &sc->sc_flow_group[group]; 5910 mbin->cmd_table_id = htobe32(mfg->g_table); 5911 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5912 5913 mcx_cmdq_mboxes_sign(&mxm, 2); 5914 mcx_cmdq_post(sc, cqe, 0); 5915 error = mcx_cmdq_poll(sc, cqe, 1000); 5916 if (error != 0) { 5917 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 5918 goto free; 5919 } 5920 if (mcx_cmdq_verify(cqe) != 0) { 5921 printf("%s: delete flow table entry command corrupt\n", 5922 DEVNAME(sc)); 5923 goto free; 5924 } 5925 5926 out = mcx_cmdq_out(cqe); 5927 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5928 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 5929 DEVNAME(sc), group, index, out->cmd_status, 5930 betoh32(out->cmd_syndrome)); 5931 error = -1; 5932 goto free; 5933 } 5934 5935 free: 5936 mcx_dmamem_free(sc, &mxm); 5937 return (error); 5938 } 5939 5940 #if 0 5941 int 5942 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 5943 { 5944 struct mcx_dmamem mxm; 5945 struct mcx_cmdq_entry *cqe; 5946 struct mcx_cmd_query_flow_table_in *in; 5947 struct mcx_cmd_query_flow_table_mb_in *mbin; 5948 struct mcx_cmd_query_flow_table_out *out; 5949 struct mcx_cmd_query_flow_table_mb_out *mbout; 5950 uint8_t token = mcx_cmdq_token(sc); 5951 int error; 5952 int i; 5953 uint8_t *dump; 5954 5955 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5956 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5957 sizeof(*out) + sizeof(*mbout) + 16, token); 5958 5959 in = mcx_cmdq_in(cqe); 5960 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 5961 in->cmd_op_mod = htobe16(0); 5962 5963 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5964 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 5965 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5966 &cqe->cq_output_ptr, token) != 0) { 5967 printf(", unable to allocate query flow table mailboxes\n"); 5968 return (-1); 5969 } 5970 cqe->cq_input_ptr = cqe->cq_output_ptr; 5971 5972 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5973 mbin->cmd_table_type = 0; 5974 mbin->cmd_table_id = htobe32(flow_table_id); 5975 5976 mcx_cmdq_mboxes_sign(&mxm, 1); 5977 5978 mcx_cmdq_post(sc, cqe, 0); 5979 error = mcx_cmdq_poll(sc, cqe, 1000); 5980 if (error != 0) { 5981 printf("%s: query flow table timeout\n", DEVNAME(sc)); 5982 goto free; 5983 } 5984 error = mcx_cmdq_verify(cqe); 5985 if (error != 0) { 5986 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 5987 goto free; 5988 } 5989 5990 out = mcx_cmdq_out(cqe); 5991 switch (out->cmd_status) { 5992 case MCX_CQ_STATUS_OK: 5993 break; 5994 default: 5995 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 5996 out->cmd_status, betoh32(out->cmd_syndrome)); 5997 error = -1; 5998 goto free; 5999 } 6000 6001 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 6002 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6003 dump = (uint8_t *)mbout + 8; 6004 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 6005 printf("%.2x ", dump[i]); 6006 if (i % 16 == 15) 6007 printf("\n"); 6008 } 6009 free: 6010 mcx_cq_mboxes_free(sc, &mxm); 6011 return (error); 6012 } 6013 int 6014 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 6015 { 6016 struct mcx_dmamem mxm; 6017 struct mcx_cmdq_entry *cqe; 6018 struct mcx_cmd_query_flow_table_entry_in *in; 6019 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 6020 struct mcx_cmd_query_flow_table_entry_out *out; 6021 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 6022 uint8_t token = mcx_cmdq_token(sc); 6023 int error; 6024 int i; 6025 uint8_t *dump; 6026 6027 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6028 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6029 sizeof(*out) + sizeof(*mbout) + 16, token); 6030 6031 in = mcx_cmdq_in(cqe); 6032 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 6033 in->cmd_op_mod = htobe16(0); 6034 6035 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6036 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6037 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6038 &cqe->cq_output_ptr, token) != 0) { 6039 printf(", unable to allocate " 6040 "query flow table entry mailboxes\n"); 6041 return (-1); 6042 } 6043 cqe->cq_input_ptr = cqe->cq_output_ptr; 6044 6045 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6046 mbin->cmd_table_type = 0; 6047 mbin->cmd_table_id = htobe32(flow_table_id); 6048 mbin->cmd_flow_index = htobe32(index); 6049 6050 mcx_cmdq_mboxes_sign(&mxm, 1); 6051 6052 mcx_cmdq_post(sc, cqe, 0); 6053 error = mcx_cmdq_poll(sc, cqe, 1000); 6054 if (error != 0) { 6055 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 6056 goto free; 6057 } 6058 error = mcx_cmdq_verify(cqe); 6059 if (error != 0) { 6060 printf("%s: query flow table entry reply corrupt\n", 6061 DEVNAME(sc)); 6062 goto free; 6063 } 6064 6065 out = mcx_cmdq_out(cqe); 6066 switch (out->cmd_status) { 6067 case MCX_CQ_STATUS_OK: 6068 break; 6069 default: 6070 printf("%s: query flow table entry failed (%x/%x)\n", 6071 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6072 error = -1; 6073 goto free; 6074 } 6075 6076 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 6077 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6078 dump = (uint8_t *)mbout; 6079 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6080 printf("%.2x ", dump[i]); 6081 if (i % 16 == 15) 6082 printf("\n"); 6083 } 6084 6085 free: 6086 mcx_cq_mboxes_free(sc, &mxm); 6087 return (error); 6088 } 6089 6090 int 6091 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 6092 { 6093 struct mcx_dmamem mxm; 6094 struct mcx_cmdq_entry *cqe; 6095 struct mcx_cmd_query_flow_group_in *in; 6096 struct mcx_cmd_query_flow_group_mb_in *mbin; 6097 struct mcx_cmd_query_flow_group_out *out; 6098 struct mcx_cmd_query_flow_group_mb_out *mbout; 6099 uint8_t token = mcx_cmdq_token(sc); 6100 int error; 6101 int i; 6102 uint8_t *dump; 6103 6104 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6105 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6106 sizeof(*out) + sizeof(*mbout) + 16, token); 6107 6108 in = mcx_cmdq_in(cqe); 6109 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 6110 in->cmd_op_mod = htobe16(0); 6111 6112 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6113 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6114 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6115 &cqe->cq_output_ptr, token) != 0) { 6116 printf(", unable to allocate query flow group mailboxes\n"); 6117 return (-1); 6118 } 6119 cqe->cq_input_ptr = cqe->cq_output_ptr; 6120 6121 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6122 mbin->cmd_table_type = 0; 6123 mbin->cmd_table_id = htobe32(flow_table_id); 6124 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 6125 6126 mcx_cmdq_mboxes_sign(&mxm, 1); 6127 6128 mcx_cmdq_post(sc, cqe, 0); 6129 error = mcx_cmdq_poll(sc, cqe, 1000); 6130 if (error != 0) { 6131 printf("%s: query flow group timeout\n", DEVNAME(sc)); 6132 goto free; 6133 } 6134 error = mcx_cmdq_verify(cqe); 6135 if (error != 0) { 6136 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 6137 goto free; 6138 } 6139 6140 out = mcx_cmdq_out(cqe); 6141 switch (out->cmd_status) { 6142 case MCX_CQ_STATUS_OK: 6143 break; 6144 default: 6145 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 6146 out->cmd_status, betoh32(out->cmd_syndrome)); 6147 error = -1; 6148 goto free; 6149 } 6150 6151 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 6152 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6153 dump = (uint8_t *)mbout; 6154 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6155 printf("%.2x ", dump[i]); 6156 if (i % 16 == 15) 6157 printf("\n"); 6158 } 6159 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 6160 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6161 printf("%.2x ", dump[i]); 6162 if (i % 16 == 15) 6163 printf("\n"); 6164 } 6165 6166 free: 6167 mcx_cq_mboxes_free(sc, &mxm); 6168 return (error); 6169 } 6170 6171 static int 6172 mcx_dump_counters(struct mcx_softc *sc) 6173 { 6174 struct mcx_dmamem mxm; 6175 struct mcx_cmdq_entry *cqe; 6176 struct mcx_cmd_query_vport_counters_in *in; 6177 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6178 struct mcx_cmd_query_vport_counters_out *out; 6179 struct mcx_nic_vport_counters *counters; 6180 int error, token; 6181 6182 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6183 token = mcx_cmdq_token(sc); 6184 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6185 sizeof(*out) + sizeof(*counters), token); 6186 6187 in = mcx_cmdq_in(cqe); 6188 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6189 in->cmd_op_mod = htobe16(0); 6190 6191 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6192 &cqe->cq_output_ptr, token) != 0) { 6193 printf(", unable to allocate " 6194 "query nic vport counters mailboxen\n"); 6195 return (-1); 6196 } 6197 cqe->cq_input_ptr = cqe->cq_output_ptr; 6198 6199 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6200 mbin->cmd_clear = 0x80; 6201 6202 mcx_cmdq_mboxes_sign(&mxm, 1); 6203 mcx_cmdq_post(sc, cqe, 0); 6204 6205 error = mcx_cmdq_poll(sc, cqe, 1000); 6206 if (error != 0) { 6207 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6208 goto free; 6209 } 6210 if (mcx_cmdq_verify(cqe) != 0) { 6211 printf("%s: query nic vport counters command corrupt\n", 6212 DEVNAME(sc)); 6213 goto free; 6214 } 6215 6216 out = mcx_cmdq_out(cqe); 6217 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6218 printf("%s: query nic vport counters failed (%x, %x)\n", 6219 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6220 error = -1; 6221 goto free; 6222 } 6223 6224 counters = (struct mcx_nic_vport_counters *) 6225 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6226 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6227 counters->rx_ucast.packets + counters->tx_ucast.packets + 6228 counters->rx_err.packets + counters->tx_err.packets) 6229 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6230 DEVNAME(sc), 6231 betoh64(counters->tx_err.packets), 6232 betoh64(counters->rx_err.packets), 6233 betoh64(counters->tx_ucast.packets), 6234 betoh64(counters->rx_ucast.packets), 6235 betoh64(counters->tx_bcast.packets), 6236 betoh64(counters->rx_bcast.packets)); 6237 free: 6238 mcx_dmamem_free(sc, &mxm); 6239 6240 return (error); 6241 } 6242 6243 static int 6244 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6245 { 6246 struct mcx_dmamem mxm; 6247 struct mcx_cmdq_entry *cqe; 6248 struct mcx_cmd_query_flow_counter_in *in; 6249 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6250 struct mcx_cmd_query_flow_counter_out *out; 6251 struct mcx_counter *counters; 6252 int error, token; 6253 6254 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6255 token = mcx_cmdq_token(sc); 6256 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6257 sizeof(*counters), token); 6258 6259 in = mcx_cmdq_in(cqe); 6260 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6261 in->cmd_op_mod = htobe16(0); 6262 6263 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6264 &cqe->cq_output_ptr, token) != 0) { 6265 printf(", unable to allocate query flow counter mailboxen\n"); 6266 return (-1); 6267 } 6268 cqe->cq_input_ptr = cqe->cq_output_ptr; 6269 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6270 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6271 mbin->cmd_clear = 0x80; 6272 6273 mcx_cmdq_mboxes_sign(&mxm, 1); 6274 mcx_cmdq_post(sc, cqe, 0); 6275 6276 error = mcx_cmdq_poll(sc, cqe, 1000); 6277 if (error != 0) { 6278 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6279 goto free; 6280 } 6281 if (mcx_cmdq_verify(cqe) != 0) { 6282 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6283 goto free; 6284 } 6285 6286 out = mcx_cmdq_out(cqe); 6287 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6288 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6289 out->cmd_status, betoh32(out->cmd_syndrome)); 6290 error = -1; 6291 goto free; 6292 } 6293 6294 counters = (struct mcx_counter *) 6295 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6296 if (counters->packets) 6297 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6298 betoh64(counters->packets)); 6299 free: 6300 mcx_dmamem_free(sc, &mxm); 6301 6302 return (error); 6303 } 6304 6305 #endif 6306 6307 #if NKSTAT > 0 6308 6309 int 6310 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx) 6311 { 6312 struct mcx_dmamem mxm; 6313 struct mcx_cmdq_entry *cqe; 6314 struct mcx_cmd_query_rq_in *in; 6315 struct mcx_cmd_query_rq_out *out; 6316 struct mcx_cmd_query_rq_mb_out *mbout; 6317 uint8_t token = mcx_cmdq_token(sc); 6318 int error; 6319 6320 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6321 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6322 token); 6323 6324 in = mcx_cmdq_in(cqe); 6325 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 6326 in->cmd_op_mod = htobe16(0); 6327 in->cmd_rqn = htobe32(rx->rx_rqn); 6328 6329 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6330 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6331 &cqe->cq_output_ptr, token) != 0) { 6332 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc)); 6333 return (-1); 6334 } 6335 6336 mcx_cmdq_mboxes_sign(&mxm, 1); 6337 6338 mcx_cmdq_post(sc, cqe, 0); 6339 error = mcx_cmdq_poll(sc, cqe, 1000); 6340 if (error != 0) { 6341 printf("%s: query rq timeout\n", DEVNAME(sc)); 6342 goto free; 6343 } 6344 error = mcx_cmdq_verify(cqe); 6345 if (error != 0) { 6346 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 6347 goto free; 6348 } 6349 6350 out = mcx_cmdq_out(cqe); 6351 switch (out->cmd_status) { 6352 case MCX_CQ_STATUS_OK: 6353 break; 6354 default: 6355 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6356 out->cmd_status, betoh32(out->cmd_syndrome)); 6357 error = -1; 6358 goto free; 6359 } 6360 6361 mbout = (struct mcx_cmd_query_rq_mb_out *) 6362 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6363 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx)); 6364 6365 free: 6366 mcx_cq_mboxes_free(sc, &mxm); 6367 return (error); 6368 } 6369 6370 int 6371 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx) 6372 { 6373 struct mcx_dmamem mxm; 6374 struct mcx_cmdq_entry *cqe; 6375 struct mcx_cmd_query_sq_in *in; 6376 struct mcx_cmd_query_sq_out *out; 6377 struct mcx_cmd_query_sq_mb_out *mbout; 6378 uint8_t token = mcx_cmdq_token(sc); 6379 int error; 6380 6381 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6382 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6383 token); 6384 6385 in = mcx_cmdq_in(cqe); 6386 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6387 in->cmd_op_mod = htobe16(0); 6388 in->cmd_sqn = htobe32(tx->tx_sqn); 6389 6390 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6391 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6392 &cqe->cq_output_ptr, token) != 0) { 6393 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc)); 6394 return (-1); 6395 } 6396 6397 mcx_cmdq_mboxes_sign(&mxm, 1); 6398 6399 mcx_cmdq_post(sc, cqe, 0); 6400 error = mcx_cmdq_poll(sc, cqe, 1000); 6401 if (error != 0) { 6402 printf("%s: query sq timeout\n", DEVNAME(sc)); 6403 goto free; 6404 } 6405 error = mcx_cmdq_verify(cqe); 6406 if (error != 0) { 6407 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6408 goto free; 6409 } 6410 6411 out = mcx_cmdq_out(cqe); 6412 switch (out->cmd_status) { 6413 case MCX_CQ_STATUS_OK: 6414 break; 6415 default: 6416 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6417 out->cmd_status, betoh32(out->cmd_syndrome)); 6418 error = -1; 6419 goto free; 6420 } 6421 6422 mbout = (struct mcx_cmd_query_sq_mb_out *) 6423 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6424 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx)); 6425 6426 free: 6427 mcx_cq_mboxes_free(sc, &mxm); 6428 return (error); 6429 } 6430 6431 int 6432 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx) 6433 { 6434 struct mcx_dmamem mxm; 6435 struct mcx_cmdq_entry *cqe; 6436 struct mcx_cmd_query_cq_in *in; 6437 struct mcx_cmd_query_cq_out *out; 6438 struct mcx_cq_ctx *ctx; 6439 uint8_t token = mcx_cmdq_token(sc); 6440 int error; 6441 6442 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6443 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6444 token); 6445 6446 in = mcx_cmdq_in(cqe); 6447 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ); 6448 in->cmd_op_mod = htobe16(0); 6449 in->cmd_cqn = htobe32(cq->cq_n); 6450 6451 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6452 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6453 &cqe->cq_output_ptr, token) != 0) { 6454 printf("%s: unable to allocate query cq mailboxes\n", DEVNAME(sc)); 6455 return (-1); 6456 } 6457 6458 mcx_cmdq_mboxes_sign(&mxm, 1); 6459 6460 mcx_cmdq_post(sc, cqe, 0); 6461 error = mcx_cmdq_poll(sc, cqe, 1000); 6462 if (error != 0) { 6463 printf("%s: query cq timeout\n", DEVNAME(sc)); 6464 goto free; 6465 } 6466 error = mcx_cmdq_verify(cqe); 6467 if (error != 0) { 6468 printf("%s: query cq reply corrupt\n", DEVNAME(sc)); 6469 goto free; 6470 } 6471 6472 out = mcx_cmdq_out(cqe); 6473 switch (out->cmd_status) { 6474 case MCX_CQ_STATUS_OK: 6475 break; 6476 default: 6477 printf("%s: query cq failed (%x/%x)\n", DEVNAME(sc), 6478 out->cmd_status, betoh32(out->cmd_syndrome)); 6479 error = -1; 6480 goto free; 6481 } 6482 6483 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6484 memcpy(cq_ctx, ctx, sizeof(*cq_ctx)); 6485 free: 6486 mcx_cq_mboxes_free(sc, &mxm); 6487 return (error); 6488 } 6489 6490 int 6491 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx) 6492 { 6493 struct mcx_dmamem mxm; 6494 struct mcx_cmdq_entry *cqe; 6495 struct mcx_cmd_query_eq_in *in; 6496 struct mcx_cmd_query_eq_out *out; 6497 struct mcx_eq_ctx *ctx; 6498 uint8_t token = mcx_cmdq_token(sc); 6499 int error; 6500 6501 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6502 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6503 token); 6504 6505 in = mcx_cmdq_in(cqe); 6506 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ); 6507 in->cmd_op_mod = htobe16(0); 6508 in->cmd_eqn = htobe32(eq->eq_n); 6509 6510 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6511 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6512 &cqe->cq_output_ptr, token) != 0) { 6513 printf("%s: unable to allocate query eq mailboxes\n", DEVNAME(sc)); 6514 return (-1); 6515 } 6516 6517 mcx_cmdq_mboxes_sign(&mxm, 1); 6518 6519 mcx_cmdq_post(sc, cqe, 0); 6520 error = mcx_cmdq_poll(sc, cqe, 1000); 6521 if (error != 0) { 6522 printf("%s: query eq timeout\n", DEVNAME(sc)); 6523 goto free; 6524 } 6525 error = mcx_cmdq_verify(cqe); 6526 if (error != 0) { 6527 printf("%s: query eq reply corrupt\n", DEVNAME(sc)); 6528 goto free; 6529 } 6530 6531 out = mcx_cmdq_out(cqe); 6532 switch (out->cmd_status) { 6533 case MCX_CQ_STATUS_OK: 6534 break; 6535 default: 6536 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc), 6537 out->cmd_status, betoh32(out->cmd_syndrome)); 6538 error = -1; 6539 goto free; 6540 } 6541 6542 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6543 memcpy(eq_ctx, ctx, sizeof(*eq_ctx)); 6544 free: 6545 mcx_cq_mboxes_free(sc, &mxm); 6546 return (error); 6547 } 6548 6549 #endif /* NKSTAT > 0 */ 6550 6551 static inline unsigned int 6552 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots) 6553 { 6554 struct mcx_rq_entry *ring, *rqe; 6555 struct mcx_slot *ms; 6556 struct mbuf *m; 6557 uint slot, p, fills; 6558 6559 ring = MCX_DMA_KVA(&rx->rx_rq_mem); 6560 p = rx->rx_prod; 6561 6562 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6563 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 6564 6565 for (fills = 0; fills < nslots; fills++) { 6566 slot = p % (1 << MCX_LOG_RQ_SIZE); 6567 6568 ms = &rx->rx_slots[slot]; 6569 rqe = &ring[slot]; 6570 6571 m = MCLGETL(NULL, M_DONTWAIT, sc->sc_rxbufsz); 6572 if (m == NULL) 6573 break; 6574 6575 m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz); 6576 m->m_data += ETHER_ALIGN; 6577 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6578 6579 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6580 BUS_DMA_NOWAIT) != 0) { 6581 m_freem(m); 6582 break; 6583 } 6584 ms->ms_m = m; 6585 6586 htobem32(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len); 6587 htobem64(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr); 6588 htobem32(&rqe->rqe_lkey, sc->sc_lkey); 6589 6590 p++; 6591 } 6592 6593 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6594 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 6595 6596 rx->rx_prod = p; 6597 6598 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6599 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 6600 htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell), 6601 p & MCX_WQ_DOORBELL_MASK); 6602 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6603 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 6604 6605 return (nslots - fills); 6606 } 6607 6608 int 6609 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6610 { 6611 u_int slots; 6612 6613 slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6614 if (slots == 0) 6615 return (1); 6616 6617 slots = mcx_rx_fill_slots(sc, rx, slots); 6618 if_rxr_put(&rx->rx_rxr, slots); 6619 return (0); 6620 } 6621 6622 void 6623 mcx_refill(void *xrx) 6624 { 6625 struct mcx_rx *rx = xrx; 6626 struct mcx_softc *sc = rx->rx_softc; 6627 6628 mcx_rx_fill(sc, rx); 6629 6630 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6631 timeout_add(&rx->rx_refill, 1); 6632 } 6633 6634 static int 6635 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6636 struct mcx_cq_entry *cqe) 6637 { 6638 struct mcx_slot *ms; 6639 bus_dmamap_t map; 6640 int slot, slots; 6641 6642 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6643 6644 ms = &tx->tx_slots[slot]; 6645 map = ms->ms_map; 6646 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6647 BUS_DMASYNC_POSTWRITE); 6648 6649 slots = 1; 6650 if (map->dm_nsegs > 1) 6651 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6652 6653 bus_dmamap_unload(sc->sc_dmat, map); 6654 m_freem(ms->ms_m); 6655 ms->ms_m = NULL; 6656 6657 return (slots); 6658 } 6659 6660 static uint64_t 6661 mcx_uptime(void) 6662 { 6663 struct timespec ts; 6664 6665 nanouptime(&ts); 6666 6667 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6668 } 6669 6670 static void 6671 mcx_calibrate_first(struct mcx_softc *sc) 6672 { 6673 struct mcx_calibration *c = &sc->sc_calibration[0]; 6674 int s; 6675 6676 sc->sc_calibration_gen = 0; 6677 6678 s = splhigh(); /* crit_enter? */ 6679 c->c_ubase = mcx_uptime(); 6680 c->c_tbase = mcx_timer(sc); 6681 splx(s); 6682 c->c_ratio = 0; 6683 6684 #ifdef notyet 6685 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 6686 #endif 6687 } 6688 6689 #define MCX_TIMESTAMP_SHIFT 24 6690 6691 static void 6692 mcx_calibrate(void *arg) 6693 { 6694 struct mcx_softc *sc = arg; 6695 struct mcx_calibration *nc, *pc; 6696 uint64_t udiff, tdiff; 6697 unsigned int gen; 6698 int s; 6699 6700 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 6701 return; 6702 6703 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 6704 6705 gen = sc->sc_calibration_gen; 6706 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6707 gen++; 6708 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6709 6710 nc->c_uptime = pc->c_ubase; 6711 nc->c_timestamp = pc->c_tbase; 6712 6713 s = splhigh(); /* crit_enter? */ 6714 nc->c_ubase = mcx_uptime(); 6715 nc->c_tbase = mcx_timer(sc); 6716 splx(s); 6717 6718 udiff = nc->c_ubase - nc->c_uptime; 6719 tdiff = nc->c_tbase - nc->c_timestamp; 6720 6721 /* 6722 * udiff is the wall clock time between calibration ticks, 6723 * which should be 32 seconds or 32 billion nanoseconds. if 6724 * we squint, 1 billion nanoseconds is kind of like a 32 bit 6725 * number, so 32 billion should still have a lot of high bits 6726 * spare. we use this space by shifting the nanoseconds up 6727 * 24 bits so we have a nice big number to divide by the 6728 * number of mcx timer ticks. 6729 */ 6730 nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff; 6731 6732 membar_producer(); 6733 sc->sc_calibration_gen = gen; 6734 } 6735 6736 static int 6737 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6738 struct mcx_cq_entry *cqe, struct mbuf_list *ml, 6739 const struct mcx_calibration *c) 6740 { 6741 struct mcx_slot *ms; 6742 struct mbuf *m; 6743 uint32_t flags; 6744 int slot; 6745 6746 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6747 6748 ms = &rx->rx_slots[slot]; 6749 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, 6750 BUS_DMASYNC_POSTREAD); 6751 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6752 6753 m = ms->ms_m; 6754 ms->ms_m = NULL; 6755 6756 m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); 6757 6758 if (cqe->cq_rx_hash_type) { 6759 m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); 6760 m->m_pkthdr.csum_flags |= M_FLOWID; 6761 } 6762 6763 flags = bemtoh32(&cqe->cq_flags); 6764 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) 6765 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 6766 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) 6767 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK | 6768 M_UDP_CSUM_IN_OK; 6769 #if NVLAN > 0 6770 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6771 m->m_pkthdr.ether_vtag = (flags & 6772 MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6773 m->m_flags |= M_VLANTAG; 6774 } 6775 #endif 6776 6777 #ifdef notyet 6778 if (ISSET(sc->sc_ac.ac_if.if_flags, IFF_LINK0) && c->c_ratio) { 6779 uint64_t t = bemtoh64(&cqe->cq_timestamp); 6780 t -= c->c_timestamp; 6781 t *= c->c_ratio; 6782 t >>= MCX_TIMESTAMP_SHIFT; 6783 t += c->c_uptime; 6784 6785 m->m_pkthdr.ph_timestamp = t; 6786 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6787 } 6788 #endif 6789 6790 ml_enqueue(ml, m); 6791 6792 return (1); 6793 } 6794 6795 static struct mcx_cq_entry * 6796 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6797 { 6798 struct mcx_cq_entry *cqe; 6799 int next; 6800 6801 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 6802 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 6803 6804 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 6805 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 6806 return (&cqe[next]); 6807 } 6808 6809 return (NULL); 6810 } 6811 6812 static void 6813 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 6814 { 6815 struct mcx_cq_doorbell *db; 6816 bus_size_t offset; 6817 uint32_t val; 6818 uint64_t uval; 6819 6820 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 6821 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6822 6823 db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell); 6824 6825 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6826 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE); 6827 6828 htobem32(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6829 htobem32(&db->db_arm_ci, val); 6830 6831 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6832 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE); 6833 6834 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL; 6835 6836 uval = (uint64_t)val << 32; 6837 uval |= cq->cq_n; 6838 6839 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval)); 6840 mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE); 6841 } 6842 6843 void 6844 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 6845 { 6846 struct mcx_rx *rx = &q->q_rx; 6847 struct mcx_tx *tx = &q->q_tx; 6848 const struct mcx_calibration *c; 6849 unsigned int gen; 6850 struct mcx_cq_entry *cqe; 6851 uint8_t *cqp; 6852 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 6853 int rxfree, txfree; 6854 6855 gen = sc->sc_calibration_gen; 6856 membar_consumer(); 6857 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6858 6859 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 6860 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 6861 6862 rxfree = 0; 6863 txfree = 0; 6864 while ((cqe = mcx_next_cq_entry(sc, cq))) { 6865 uint8_t opcode; 6866 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 6867 switch (opcode) { 6868 case MCX_CQ_ENTRY_OPCODE_REQ: 6869 txfree += mcx_process_txeof(sc, tx, cqe); 6870 break; 6871 case MCX_CQ_ENTRY_OPCODE_SEND: 6872 rxfree += mcx_process_rx(sc, rx, cqe, &ml, c); 6873 break; 6874 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 6875 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 6876 cqp = (uint8_t *)cqe; 6877 /* printf("%s: cq completion error: %x\n", 6878 DEVNAME(sc), cqp[0x37]); */ 6879 break; 6880 6881 default: 6882 /* printf("%s: cq completion opcode %x??\n", 6883 DEVNAME(sc), opcode); */ 6884 break; 6885 } 6886 6887 cq->cq_cons++; 6888 } 6889 6890 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 6891 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 6892 6893 cq->cq_count++; 6894 mcx_arm_cq(sc, cq, q->q_uar); 6895 6896 if (rxfree > 0) { 6897 if_rxr_put(&rx->rx_rxr, rxfree); 6898 if (ifiq_input(rx->rx_ifiq, &ml)) 6899 if_rxr_livelocked(&rx->rx_rxr); 6900 6901 mcx_rx_fill(sc, rx); 6902 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6903 timeout_add(&rx->rx_refill, 1); 6904 } 6905 if (txfree > 0) { 6906 tx->tx_cons += txfree; 6907 if (ifq_is_oactive(tx->tx_ifq)) 6908 ifq_restart(tx->tx_ifq); 6909 } 6910 } 6911 6912 6913 static void 6914 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 6915 { 6916 bus_size_t offset; 6917 uint32_t val; 6918 6919 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 6920 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 6921 6922 mcx_wr(sc, offset, val); 6923 mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE); 6924 } 6925 6926 static struct mcx_eq_entry * 6927 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 6928 { 6929 struct mcx_eq_entry *eqe; 6930 int next; 6931 6932 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 6933 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 6934 if ((eqe[next].eq_owner & 1) == 6935 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 6936 eq->eq_cons++; 6937 return (&eqe[next]); 6938 } 6939 return (NULL); 6940 } 6941 6942 int 6943 mcx_admin_intr(void *xsc) 6944 { 6945 struct mcx_softc *sc = (struct mcx_softc *)xsc; 6946 struct mcx_eq *eq = &sc->sc_admin_eq; 6947 struct mcx_eq_entry *eqe; 6948 6949 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6950 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 6951 6952 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 6953 switch (eqe->eq_event_type) { 6954 case MCX_EVENT_TYPE_LAST_WQE: 6955 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 6956 break; 6957 6958 case MCX_EVENT_TYPE_CQ_ERROR: 6959 /* printf("%s: cq error\n", DEVNAME(sc)); */ 6960 break; 6961 6962 case MCX_EVENT_TYPE_CMD_COMPLETION: 6963 /* wakeup probably */ 6964 break; 6965 6966 case MCX_EVENT_TYPE_PORT_CHANGE: 6967 task_add(systq, &sc->sc_port_change); 6968 break; 6969 6970 default: 6971 /* printf("%s: something happened\n", DEVNAME(sc)); */ 6972 break; 6973 } 6974 } 6975 6976 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6977 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 6978 6979 mcx_arm_eq(sc, eq, sc->sc_uar); 6980 6981 return (1); 6982 } 6983 6984 int 6985 mcx_cq_intr(void *xq) 6986 { 6987 struct mcx_queues *q = (struct mcx_queues *)xq; 6988 struct mcx_softc *sc = q->q_sc; 6989 struct mcx_eq *eq = &q->q_eq; 6990 struct mcx_eq_entry *eqe; 6991 int cqn; 6992 6993 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6994 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 6995 6996 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 6997 switch (eqe->eq_event_type) { 6998 case MCX_EVENT_TYPE_COMPLETION: 6999 cqn = betoh32(eqe->eq_event_data[6]); 7000 if (cqn == q->q_cq.cq_n) 7001 mcx_process_cq(sc, q, &q->q_cq); 7002 break; 7003 } 7004 } 7005 7006 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7007 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 7008 7009 mcx_arm_eq(sc, eq, q->q_uar); 7010 7011 return (1); 7012 } 7013 7014 static void 7015 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 7016 int total) 7017 { 7018 struct mcx_slot *ms; 7019 7020 int i = allocated; 7021 while (i-- > 0) { 7022 ms = &slots[i]; 7023 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 7024 if (ms->ms_m != NULL) 7025 m_freem(ms->ms_m); 7026 } 7027 free(slots, M_DEVBUF, total * sizeof(*ms)); 7028 } 7029 7030 static int 7031 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 7032 { 7033 struct mcx_rx *rx; 7034 struct mcx_tx *tx; 7035 struct mcx_slot *ms; 7036 int i; 7037 7038 rx = &q->q_rx; 7039 rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 7040 M_DEVBUF, M_WAITOK | M_ZERO); 7041 if (rx->rx_slots == NULL) { 7042 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 7043 return ENOMEM; 7044 } 7045 7046 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 7047 ms = &rx->rx_slots[i]; 7048 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 7049 sc->sc_hardmtu, 0, 7050 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7051 &ms->ms_map) != 0) { 7052 printf("%s: failed to allocate rx dma maps\n", 7053 DEVNAME(sc)); 7054 goto destroy_rx_slots; 7055 } 7056 } 7057 7058 tx = &q->q_tx; 7059 tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 7060 M_DEVBUF, M_WAITOK | M_ZERO); 7061 if (tx->tx_slots == NULL) { 7062 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 7063 goto destroy_rx_slots; 7064 } 7065 7066 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 7067 ms = &tx->tx_slots[i]; 7068 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 7069 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 7070 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7071 &ms->ms_map) != 0) { 7072 printf("%s: failed to allocate tx dma maps\n", 7073 DEVNAME(sc)); 7074 goto destroy_tx_slots; 7075 } 7076 } 7077 7078 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 7079 q->q_eq.eq_n) != 0) 7080 goto destroy_tx_slots; 7081 7082 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 7083 != 0) 7084 goto destroy_cq; 7085 7086 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 7087 goto destroy_sq; 7088 7089 return 0; 7090 7091 destroy_sq: 7092 mcx_destroy_sq(sc, tx); 7093 destroy_cq: 7094 mcx_destroy_cq(sc, &q->q_cq); 7095 destroy_tx_slots: 7096 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 7097 tx->tx_slots = NULL; 7098 7099 i = (1 << MCX_LOG_RQ_SIZE); 7100 destroy_rx_slots: 7101 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 7102 rx->rx_slots = NULL; 7103 return ENOMEM; 7104 } 7105 7106 static int 7107 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 7108 { 7109 int i; 7110 int count; 7111 7112 count = 0; 7113 for (i = 0; i < nitems(mcx_rss_config); i++) { 7114 if (mcx_rss_config[i].flow_group == group) 7115 count++; 7116 } 7117 7118 return count; 7119 } 7120 7121 static int 7122 mcx_up(struct mcx_softc *sc) 7123 { 7124 struct ifnet *ifp = &sc->sc_ac.ac_if; 7125 struct mcx_rx *rx; 7126 struct mcx_tx *tx; 7127 int i, start, count, flow_group, flow_index; 7128 struct mcx_flow_match match_crit; 7129 struct mcx_rss_rule *rss; 7130 uint32_t dest; 7131 int rqns[MCX_MAX_QUEUES]; 7132 7133 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 7134 goto down; 7135 7136 for (i = 0; i < sc->sc_nqueues; i++) { 7137 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 7138 goto down; 7139 } 7140 } 7141 7142 /* RSS flow table and flow groups */ 7143 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 7144 &sc->sc_rss_flow_table_id) != 0) 7145 goto down; 7146 7147 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7148 sc->sc_rss_flow_table_id; 7149 7150 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 7151 memset(&match_crit, 0, sizeof(match_crit)); 7152 match_crit.mc_ethertype = 0xffff; 7153 match_crit.mc_ip_proto = 0xff; 7154 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 7155 start = 0; 7156 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 7157 if (count != 0) { 7158 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7159 MCX_FLOW_GROUP_RSS_L4, start, count, 7160 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7161 goto down; 7162 start += count; 7163 } 7164 7165 /* L3 RSS flow group (v4/v6, including fragments) */ 7166 memset(&match_crit, 0, sizeof(match_crit)); 7167 match_crit.mc_ethertype = 0xffff; 7168 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 7169 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7170 MCX_FLOW_GROUP_RSS_L3, start, count, 7171 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7172 goto down; 7173 start += count; 7174 7175 /* non-RSS flow group */ 7176 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 7177 memset(&match_crit, 0, sizeof(match_crit)); 7178 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7179 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 7180 goto down; 7181 7182 /* Root flow table, matching packets based on mac address */ 7183 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 7184 &sc->sc_mac_flow_table_id) != 0) 7185 goto down; 7186 7187 /* promisc flow group */ 7188 start = 0; 7189 memset(&match_crit, 0, sizeof(match_crit)); 7190 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7191 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 7192 goto down; 7193 sc->sc_promisc_flow_enabled = 0; 7194 start++; 7195 7196 /* all multicast flow group */ 7197 match_crit.mc_dest_mac[0] = 0x01; 7198 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7199 MCX_FLOW_GROUP_ALLMULTI, start, 1, 7200 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7201 goto down; 7202 sc->sc_allmulti_flow_enabled = 0; 7203 start++; 7204 7205 /* mac address matching flow group */ 7206 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 7207 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7208 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 7209 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7210 goto down; 7211 7212 /* flow table entries for unicast and broadcast */ 7213 start = 0; 7214 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7215 sc->sc_ac.ac_enaddr, dest) != 0) 7216 goto down; 7217 start++; 7218 7219 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7220 etherbroadcastaddr, dest) != 0) 7221 goto down; 7222 start++; 7223 7224 /* multicast entries go after that */ 7225 sc->sc_mcast_flow_base = start; 7226 7227 /* re-add any existing multicast flows */ 7228 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7229 if (sc->sc_mcast_flows[i][0] != 0) { 7230 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 7231 sc->sc_mcast_flow_base + i, 7232 sc->sc_mcast_flows[i], dest); 7233 } 7234 } 7235 7236 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 7237 goto down; 7238 7239 /* 7240 * the RQT can be any size as long as it's a power of two. 7241 * since we also restrict the number of queues to a power of two, 7242 * we can just put each rx queue in once. 7243 */ 7244 for (i = 0; i < sc->sc_nqueues; i++) 7245 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 7246 7247 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) 7248 goto down; 7249 7250 start = 0; 7251 flow_index = 0; 7252 flow_group = -1; 7253 for (i = 0; i < nitems(mcx_rss_config); i++) { 7254 rss = &mcx_rss_config[i]; 7255 if (rss->flow_group != flow_group) { 7256 flow_group = rss->flow_group; 7257 flow_index = 0; 7258 } 7259 7260 if (rss->hash_sel == 0) { 7261 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 7262 &sc->sc_tir[i]) != 0) 7263 goto down; 7264 } else { 7265 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 7266 rss->hash_sel, &sc->sc_tir[i]) != 0) 7267 goto down; 7268 } 7269 7270 if (mcx_set_flow_table_entry_proto(sc, flow_group, 7271 flow_index, rss->ethertype, rss->ip_proto, 7272 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 7273 goto down; 7274 flow_index++; 7275 } 7276 7277 for (i = 0; i < sc->sc_nqueues; i++) { 7278 struct mcx_queues *q = &sc->sc_queues[i]; 7279 rx = &q->q_rx; 7280 tx = &q->q_tx; 7281 7282 /* start the queues */ 7283 if (mcx_ready_sq(sc, tx) != 0) 7284 goto down; 7285 7286 if (mcx_ready_rq(sc, rx) != 0) 7287 goto down; 7288 7289 if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 7290 rx->rx_prod = 0; 7291 mcx_rx_fill(sc, rx); 7292 7293 tx->tx_cons = 0; 7294 tx->tx_prod = 0; 7295 ifq_clr_oactive(tx->tx_ifq); 7296 } 7297 7298 mcx_calibrate_first(sc); 7299 7300 SET(ifp->if_flags, IFF_RUNNING); 7301 7302 return ENETRESET; 7303 down: 7304 mcx_down(sc); 7305 return ENOMEM; 7306 } 7307 7308 static void 7309 mcx_down(struct mcx_softc *sc) 7310 { 7311 struct ifnet *ifp = &sc->sc_ac.ac_if; 7312 struct mcx_rss_rule *rss; 7313 int group, i, flow_group, flow_index; 7314 7315 CLR(ifp->if_flags, IFF_RUNNING); 7316 7317 /* 7318 * delete flow table entries first, so no packets can arrive 7319 * after the barriers 7320 */ 7321 if (sc->sc_promisc_flow_enabled) 7322 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 7323 if (sc->sc_allmulti_flow_enabled) 7324 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 7325 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 7326 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 7327 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7328 if (sc->sc_mcast_flows[i][0] != 0) { 7329 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 7330 sc->sc_mcast_flow_base + i); 7331 } 7332 } 7333 7334 flow_group = -1; 7335 flow_index = 0; 7336 for (i = 0; i < nitems(mcx_rss_config); i++) { 7337 rss = &mcx_rss_config[i]; 7338 if (rss->flow_group != flow_group) { 7339 flow_group = rss->flow_group; 7340 flow_index = 0; 7341 } 7342 7343 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 7344 7345 mcx_destroy_tir(sc, sc->sc_tir[i]); 7346 sc->sc_tir[i] = 0; 7347 7348 flow_index++; 7349 } 7350 intr_barrier(sc->sc_ihc); 7351 for (i = 0; i < sc->sc_nqueues; i++) { 7352 struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; 7353 ifq_barrier(ifq); 7354 7355 timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill); 7356 7357 intr_barrier(sc->sc_queues[i].q_ihc); 7358 } 7359 7360 timeout_del_barrier(&sc->sc_calibrate); 7361 7362 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 7363 if (sc->sc_flow_group[group].g_id != -1) 7364 mcx_destroy_flow_group(sc, group); 7365 } 7366 7367 if (sc->sc_mac_flow_table_id != -1) { 7368 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 7369 sc->sc_mac_flow_table_id = -1; 7370 } 7371 if (sc->sc_rss_flow_table_id != -1) { 7372 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 7373 sc->sc_rss_flow_table_id = -1; 7374 } 7375 if (sc->sc_rqt != -1) { 7376 mcx_destroy_rqt(sc, sc->sc_rqt); 7377 sc->sc_rqt = -1; 7378 } 7379 7380 for (i = 0; i < sc->sc_nqueues; i++) { 7381 struct mcx_queues *q = &sc->sc_queues[i]; 7382 struct mcx_rx *rx = &q->q_rx; 7383 struct mcx_tx *tx = &q->q_tx; 7384 struct mcx_cq *cq = &q->q_cq; 7385 7386 if (rx->rx_rqn != 0) 7387 mcx_destroy_rq(sc, rx); 7388 7389 if (tx->tx_sqn != 0) 7390 mcx_destroy_sq(sc, tx); 7391 7392 if (tx->tx_slots != NULL) { 7393 mcx_free_slots(sc, tx->tx_slots, 7394 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7395 tx->tx_slots = NULL; 7396 } 7397 if (rx->rx_slots != NULL) { 7398 mcx_free_slots(sc, rx->rx_slots, 7399 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7400 rx->rx_slots = NULL; 7401 } 7402 7403 if (cq->cq_n != 0) 7404 mcx_destroy_cq(sc, cq); 7405 } 7406 if (sc->sc_tis != 0) { 7407 mcx_destroy_tis(sc, sc->sc_tis); 7408 sc->sc_tis = 0; 7409 } 7410 } 7411 7412 static int 7413 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 7414 { 7415 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7416 struct ifreq *ifr = (struct ifreq *)data; 7417 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7418 int s, i, error = 0; 7419 uint32_t dest; 7420 7421 s = splnet(); 7422 switch (cmd) { 7423 case SIOCSIFADDR: 7424 ifp->if_flags |= IFF_UP; 7425 /* FALLTHROUGH */ 7426 7427 case SIOCSIFFLAGS: 7428 if (ISSET(ifp->if_flags, IFF_UP)) { 7429 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7430 error = ENETRESET; 7431 else 7432 error = mcx_up(sc); 7433 } else { 7434 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7435 mcx_down(sc); 7436 } 7437 break; 7438 7439 case SIOCGIFMEDIA: 7440 case SIOCSIFMEDIA: 7441 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 7442 break; 7443 7444 case SIOCGIFSFFPAGE: 7445 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 7446 break; 7447 7448 case SIOCGIFRXR: 7449 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 7450 break; 7451 7452 case SIOCADDMULTI: 7453 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 7454 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7455 if (error != 0) 7456 return (error); 7457 7458 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7459 sc->sc_rss_flow_table_id; 7460 7461 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7462 if (sc->sc_mcast_flows[i][0] == 0) { 7463 memcpy(sc->sc_mcast_flows[i], addrlo, 7464 ETHER_ADDR_LEN); 7465 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7466 mcx_set_flow_table_entry_mac(sc, 7467 MCX_FLOW_GROUP_MAC, 7468 sc->sc_mcast_flow_base + i, 7469 sc->sc_mcast_flows[i], dest); 7470 } 7471 break; 7472 } 7473 } 7474 7475 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7476 if (i == MCX_NUM_MCAST_FLOWS) { 7477 SET(ifp->if_flags, IFF_ALLMULTI); 7478 sc->sc_extra_mcast++; 7479 error = ENETRESET; 7480 } 7481 7482 if (sc->sc_ac.ac_multirangecnt > 0) { 7483 SET(ifp->if_flags, IFF_ALLMULTI); 7484 error = ENETRESET; 7485 } 7486 } 7487 } 7488 break; 7489 7490 case SIOCDELMULTI: 7491 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 7492 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7493 if (error != 0) 7494 return (error); 7495 7496 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7497 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7498 ETHER_ADDR_LEN) == 0) { 7499 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7500 mcx_delete_flow_table_entry(sc, 7501 MCX_FLOW_GROUP_MAC, 7502 sc->sc_mcast_flow_base + i); 7503 } 7504 sc->sc_mcast_flows[i][0] = 0; 7505 break; 7506 } 7507 } 7508 7509 if (i == MCX_NUM_MCAST_FLOWS) 7510 sc->sc_extra_mcast--; 7511 7512 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7513 (sc->sc_extra_mcast == 0) && 7514 (sc->sc_ac.ac_multirangecnt == 0)) { 7515 CLR(ifp->if_flags, IFF_ALLMULTI); 7516 error = ENETRESET; 7517 } 7518 } 7519 break; 7520 7521 default: 7522 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 7523 } 7524 7525 if (error == ENETRESET) { 7526 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7527 (IFF_UP | IFF_RUNNING)) 7528 mcx_iff(sc); 7529 error = 0; 7530 } 7531 splx(s); 7532 7533 return (error); 7534 } 7535 7536 static int 7537 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7538 { 7539 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7540 struct mcx_reg_mcia mcia; 7541 struct mcx_reg_pmlp pmlp; 7542 int offset, error; 7543 7544 /* get module number */ 7545 memset(&pmlp, 0, sizeof(pmlp)); 7546 pmlp.rp_local_port = 1; 7547 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7548 sizeof(pmlp)); 7549 if (error != 0) { 7550 printf("%s: unable to get eeprom module number\n", 7551 DEVNAME(sc)); 7552 return error; 7553 } 7554 7555 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7556 memset(&mcia, 0, sizeof(mcia)); 7557 mcia.rm_l = 0; 7558 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 7559 MCX_PMLP_MODULE_NUM_MASK; 7560 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7561 mcia.rm_page_num = sff->sff_page; 7562 mcia.rm_dev_addr = htobe16(offset); 7563 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7564 7565 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7566 &mcia, sizeof(mcia)); 7567 if (error != 0) { 7568 printf("%s: unable to read eeprom at %x\n", 7569 DEVNAME(sc), offset); 7570 return error; 7571 } 7572 7573 memcpy(sff->sff_data + offset, mcia.rm_data, 7574 MCX_MCIA_EEPROM_BYTES); 7575 } 7576 7577 return 0; 7578 } 7579 7580 static int 7581 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 7582 { 7583 struct if_rxring_info *ifrs; 7584 unsigned int i; 7585 int error; 7586 7587 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP, 7588 M_WAITOK|M_ZERO|M_CANFAIL); 7589 if (ifrs == NULL) 7590 return (ENOMEM); 7591 7592 for (i = 0; i < sc->sc_nqueues; i++) { 7593 struct mcx_rx *rx = &sc->sc_queues[i].q_rx; 7594 struct if_rxring_info *ifr = &ifrs[i]; 7595 7596 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 7597 ifr->ifr_size = sc->sc_hardmtu; 7598 ifr->ifr_info = rx->rx_rxr; 7599 } 7600 7601 error = if_rxr_info_ioctl(ifri, i, ifrs); 7602 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 7603 7604 return (error); 7605 } 7606 7607 int 7608 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7609 { 7610 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7611 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7612 case 0: 7613 break; 7614 7615 case EFBIG: 7616 if (m_defrag(m, M_DONTWAIT) == 0 && 7617 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7618 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7619 break; 7620 7621 default: 7622 return (1); 7623 } 7624 7625 ms->ms_m = m; 7626 return (0); 7627 } 7628 7629 static void 7630 mcx_start(struct ifqueue *ifq) 7631 { 7632 struct mcx_tx *tx = ifq->ifq_softc; 7633 struct ifnet *ifp = ifq->ifq_if; 7634 struct mcx_softc *sc = ifp->if_softc; 7635 struct mcx_sq_entry *sq, *sqe; 7636 struct mcx_sq_entry_seg *sqs; 7637 struct mcx_slot *ms; 7638 bus_dmamap_t map; 7639 struct mbuf *m; 7640 u_int idx, free, used; 7641 uint64_t *bf; 7642 uint32_t csum; 7643 size_t bf_base; 7644 int i, seg, nseg; 7645 7646 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7647 7648 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7649 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7650 7651 used = 0; 7652 bf = NULL; 7653 7654 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7655 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 7656 7657 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7658 7659 for (;;) { 7660 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7661 ifq_set_oactive(ifq); 7662 break; 7663 } 7664 7665 m = ifq_dequeue(ifq); 7666 if (m == NULL) { 7667 break; 7668 } 7669 7670 sqe = sq + idx; 7671 ms = &tx->tx_slots[idx]; 7672 memset(sqe, 0, sizeof(*sqe)); 7673 7674 /* ctrl segment */ 7675 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7676 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7677 /* always generate a completion event */ 7678 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7679 7680 /* eth segment */ 7681 csum = 0; 7682 if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 7683 csum |= MCX_SQE_L3_CSUM; 7684 if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) 7685 csum |= MCX_SQE_L4_CSUM; 7686 sqe->sqe_mss_csum = htobe32(csum); 7687 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7688 #if NVLAN > 0 7689 if (m->m_flags & M_VLANTAG) { 7690 struct ether_vlan_header *evh; 7691 evh = (struct ether_vlan_header *) 7692 &sqe->sqe_inline_headers; 7693 7694 /* slightly cheaper vlan_inject() */ 7695 m_copydata(m, 0, ETHER_HDR_LEN, (caddr_t)evh); 7696 evh->evl_proto = evh->evl_encap_proto; 7697 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7698 evh->evl_tag = htons(m->m_pkthdr.ether_vtag); 7699 7700 m_adj(m, ETHER_HDR_LEN); 7701 } else 7702 #endif 7703 { 7704 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7705 (caddr_t)sqe->sqe_inline_headers); 7706 m_adj(m, MCX_SQ_INLINE_SIZE); 7707 } 7708 7709 if (mcx_load_mbuf(sc, ms, m) != 0) { 7710 m_freem(m); 7711 ifp->if_oerrors++; 7712 continue; 7713 } 7714 bf = (uint64_t *)sqe; 7715 7716 #if NBPFILTER > 0 7717 if (ifp->if_bpf) 7718 bpf_mtap_hdr(ifp->if_bpf, 7719 (caddr_t)sqe->sqe_inline_headers, 7720 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 7721 #endif 7722 map = ms->ms_map; 7723 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7724 BUS_DMASYNC_PREWRITE); 7725 7726 sqe->sqe_ds_sq_num = 7727 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7728 (map->dm_nsegs + 3)); 7729 7730 /* data segment - first wqe has one segment */ 7731 sqs = sqe->sqe_segs; 7732 seg = 0; 7733 nseg = 1; 7734 for (i = 0; i < map->dm_nsegs; i++) { 7735 if (seg == nseg) { 7736 /* next slot */ 7737 idx++; 7738 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7739 idx = 0; 7740 tx->tx_prod++; 7741 used++; 7742 7743 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7744 seg = 0; 7745 nseg = MCX_SQ_SEGS_PER_SLOT; 7746 } 7747 sqs[seg].sqs_byte_count = 7748 htobe32(map->dm_segs[i].ds_len); 7749 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7750 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7751 seg++; 7752 } 7753 7754 idx++; 7755 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7756 idx = 0; 7757 tx->tx_prod++; 7758 used++; 7759 } 7760 7761 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7762 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 7763 7764 if (used) { 7765 bus_size_t blueflame; 7766 7767 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7768 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 7769 htobem32(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell), 7770 tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7771 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7772 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 7773 7774 /* 7775 * write the first 64 bits of the last sqe we produced 7776 * to the blue flame buffer 7777 */ 7778 7779 blueflame = bf_base + tx->tx_bf_offset; 7780 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 7781 blueflame, *bf); 7782 mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE); 7783 7784 /* next write goes to the other buffer */ 7785 tx->tx_bf_offset ^= sc->sc_bf_size; 7786 } 7787 } 7788 7789 static void 7790 mcx_watchdog(struct ifnet *ifp) 7791 { 7792 } 7793 7794 static void 7795 mcx_media_add_types(struct mcx_softc *sc) 7796 { 7797 struct mcx_reg_ptys ptys; 7798 int i; 7799 uint32_t proto_cap; 7800 7801 memset(&ptys, 0, sizeof(ptys)); 7802 ptys.rp_local_port = 1; 7803 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7804 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7805 sizeof(ptys)) != 0) { 7806 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7807 return; 7808 } 7809 7810 proto_cap = betoh32(ptys.rp_eth_proto_cap); 7811 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7812 const struct mcx_eth_proto_capability *cap; 7813 if (!ISSET(proto_cap, 1 << i)) 7814 continue; 7815 7816 cap = &mcx_eth_cap_map[i]; 7817 if (cap->cap_media == 0) 7818 continue; 7819 7820 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 7821 } 7822 } 7823 7824 static void 7825 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 7826 { 7827 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7828 struct mcx_reg_ptys ptys; 7829 int i; 7830 uint32_t proto_oper; 7831 uint64_t media_oper; 7832 7833 memset(&ptys, 0, sizeof(ptys)); 7834 ptys.rp_local_port = 1; 7835 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7836 7837 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7838 sizeof(ptys)) != 0) { 7839 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7840 return; 7841 } 7842 7843 proto_oper = betoh32(ptys.rp_eth_proto_oper); 7844 7845 media_oper = 0; 7846 7847 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7848 const struct mcx_eth_proto_capability *cap; 7849 if (!ISSET(proto_oper, 1 << i)) 7850 continue; 7851 7852 cap = &mcx_eth_cap_map[i]; 7853 7854 if (cap->cap_media != 0) 7855 media_oper = cap->cap_media; 7856 } 7857 7858 ifmr->ifm_status = IFM_AVALID; 7859 if (proto_oper != 0) { 7860 ifmr->ifm_status |= IFM_ACTIVE; 7861 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 7862 /* txpause, rxpause, duplex? */ 7863 } 7864 } 7865 7866 static int 7867 mcx_media_change(struct ifnet *ifp) 7868 { 7869 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7870 struct mcx_reg_ptys ptys; 7871 struct mcx_reg_paos paos; 7872 uint32_t media; 7873 int i, error; 7874 7875 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 7876 return EINVAL; 7877 7878 error = 0; 7879 7880 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 7881 /* read ptys to get supported media */ 7882 memset(&ptys, 0, sizeof(ptys)); 7883 ptys.rp_local_port = 1; 7884 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7885 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 7886 &ptys, sizeof(ptys)) != 0) { 7887 printf("%s: unable to read port type/speed\n", 7888 DEVNAME(sc)); 7889 return EIO; 7890 } 7891 7892 media = betoh32(ptys.rp_eth_proto_cap); 7893 } else { 7894 /* map media type */ 7895 media = 0; 7896 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7897 const struct mcx_eth_proto_capability *cap; 7898 7899 cap = &mcx_eth_cap_map[i]; 7900 if (cap->cap_media == 7901 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 7902 media = (1 << i); 7903 break; 7904 } 7905 } 7906 } 7907 7908 /* disable the port */ 7909 memset(&paos, 0, sizeof(paos)); 7910 paos.rp_local_port = 1; 7911 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 7912 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7913 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7914 sizeof(paos)) != 0) { 7915 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 7916 return EIO; 7917 } 7918 7919 memset(&ptys, 0, sizeof(ptys)); 7920 ptys.rp_local_port = 1; 7921 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7922 ptys.rp_eth_proto_admin = htobe32(media); 7923 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 7924 sizeof(ptys)) != 0) { 7925 printf("%s: unable to set port media type/speed\n", 7926 DEVNAME(sc)); 7927 error = EIO; 7928 } 7929 7930 /* re-enable the port to start negotiation */ 7931 memset(&paos, 0, sizeof(paos)); 7932 paos.rp_local_port = 1; 7933 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 7934 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7935 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7936 sizeof(paos)) != 0) { 7937 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 7938 error = EIO; 7939 } 7940 7941 return error; 7942 } 7943 7944 static void 7945 mcx_port_change(void *xsc) 7946 { 7947 struct mcx_softc *sc = xsc; 7948 struct ifnet *ifp = &sc->sc_ac.ac_if; 7949 struct mcx_reg_ptys ptys = { 7950 .rp_local_port = 1, 7951 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 7952 }; 7953 int link_state = LINK_STATE_DOWN; 7954 7955 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7956 sizeof(ptys)) == 0) { 7957 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 7958 uint64_t baudrate = 0; 7959 unsigned int i; 7960 7961 if (proto_oper != 0) 7962 link_state = LINK_STATE_FULL_DUPLEX; 7963 7964 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7965 const struct mcx_eth_proto_capability *cap; 7966 if (!ISSET(proto_oper, 1 << i)) 7967 continue; 7968 7969 cap = &mcx_eth_cap_map[i]; 7970 if (cap->cap_baudrate == 0) 7971 continue; 7972 7973 baudrate = cap->cap_baudrate; 7974 break; 7975 } 7976 7977 ifp->if_baudrate = baudrate; 7978 } 7979 7980 if (link_state != ifp->if_link_state) { 7981 ifp->if_link_state = link_state; 7982 if_link_state_change(ifp); 7983 } 7984 } 7985 7986 static inline uint32_t 7987 mcx_rd(struct mcx_softc *sc, bus_size_t r) 7988 { 7989 uint32_t word; 7990 7991 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 7992 7993 return (betoh32(word)); 7994 } 7995 7996 static inline void 7997 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 7998 { 7999 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 8000 } 8001 8002 static inline void 8003 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 8004 { 8005 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 8006 } 8007 8008 static uint64_t 8009 mcx_timer(struct mcx_softc *sc) 8010 { 8011 uint32_t hi, lo, ni; 8012 8013 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8014 for (;;) { 8015 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 8016 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 8017 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8018 8019 if (ni == hi) 8020 break; 8021 8022 hi = ni; 8023 } 8024 8025 return (((uint64_t)hi << 32) | (uint64_t)lo); 8026 } 8027 8028 static int 8029 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 8030 bus_size_t size, u_int align) 8031 { 8032 mxm->mxm_size = size; 8033 8034 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 8035 mxm->mxm_size, 0, 8036 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 8037 &mxm->mxm_map) != 0) 8038 return (1); 8039 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 8040 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 8041 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 8042 goto destroy; 8043 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 8044 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 8045 goto free; 8046 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 8047 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 8048 goto unmap; 8049 8050 return (0); 8051 unmap: 8052 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8053 free: 8054 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8055 destroy: 8056 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8057 return (1); 8058 } 8059 8060 static void 8061 mcx_dmamem_zero(struct mcx_dmamem *mxm) 8062 { 8063 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 8064 } 8065 8066 static void 8067 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 8068 { 8069 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 8070 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8071 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8072 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8073 } 8074 8075 static int 8076 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 8077 { 8078 bus_dma_segment_t *segs; 8079 bus_size_t len = pages * MCX_PAGE_SIZE; 8080 size_t seglen; 8081 8082 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 8083 if (segs == NULL) 8084 return (-1); 8085 8086 seglen = sizeof(*segs) * pages; 8087 8088 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 8089 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 8090 goto free_segs; 8091 8092 if (mhm->mhm_seg_count < pages) { 8093 size_t nseglen; 8094 8095 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 8096 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 8097 if (mhm->mhm_segs == NULL) 8098 goto free_dmamem; 8099 8100 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 8101 8102 memcpy(mhm->mhm_segs, segs, nseglen); 8103 8104 free(segs, M_DEVBUF, seglen); 8105 8106 segs = mhm->mhm_segs; 8107 seglen = nseglen; 8108 } else 8109 mhm->mhm_segs = segs; 8110 8111 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 8112 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 8113 &mhm->mhm_map) != 0) 8114 goto free_dmamem; 8115 8116 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 8117 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 8118 goto destroy; 8119 8120 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8121 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 8122 8123 mhm->mhm_npages = pages; 8124 8125 return (0); 8126 8127 destroy: 8128 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8129 free_dmamem: 8130 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8131 free_segs: 8132 free(segs, M_DEVBUF, seglen); 8133 mhm->mhm_segs = NULL; 8134 8135 return (-1); 8136 } 8137 8138 static void 8139 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 8140 { 8141 if (mhm->mhm_npages == 0) 8142 return; 8143 8144 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8145 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 8146 8147 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 8148 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8149 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8150 free(mhm->mhm_segs, M_DEVBUF, 8151 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 8152 8153 mhm->mhm_npages = 0; 8154 } 8155 8156 #if NKSTAT > 0 8157 struct mcx_ppcnt { 8158 char name[KSTAT_KV_NAMELEN]; 8159 enum kstat_kv_unit unit; 8160 }; 8161 8162 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 8163 { "Good Tx", KSTAT_KV_U_PACKETS, }, 8164 { "Good Rx", KSTAT_KV_U_PACKETS, }, 8165 { "FCS errs", KSTAT_KV_U_PACKETS, }, 8166 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8167 { "Good Tx", KSTAT_KV_U_BYTES, }, 8168 { "Good Rx", KSTAT_KV_U_BYTES, }, 8169 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 8170 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 8171 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 8172 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 8173 { "In Range Len", KSTAT_KV_U_PACKETS, }, 8174 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 8175 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 8176 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 8177 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 8178 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 8179 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 8180 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8181 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8182 }; 8183 CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 8184 8185 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 8186 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 8187 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 8188 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 8189 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 8190 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 8191 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 8192 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 8193 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 8194 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 8195 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 8196 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 8197 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 8198 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 8199 }; 8200 CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 8201 8202 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 8203 { "Drop Events", KSTAT_KV_U_PACKETS, }, 8204 { "Octets", KSTAT_KV_U_BYTES, }, 8205 { "Packets", KSTAT_KV_U_PACKETS, }, 8206 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 8207 { "Multicasts", KSTAT_KV_U_PACKETS, }, 8208 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 8209 { "Undersize", KSTAT_KV_U_PACKETS, }, 8210 { "Oversize", KSTAT_KV_U_PACKETS, }, 8211 { "Fragments", KSTAT_KV_U_PACKETS, }, 8212 { "Jabbers", KSTAT_KV_U_PACKETS, }, 8213 { "Collisions", KSTAT_KV_U_NONE, }, 8214 { "64B", KSTAT_KV_U_PACKETS, }, 8215 { "65-127B", KSTAT_KV_U_PACKETS, }, 8216 { "128-255B", KSTAT_KV_U_PACKETS, }, 8217 { "256-511B", KSTAT_KV_U_PACKETS, }, 8218 { "512-1023B", KSTAT_KV_U_PACKETS, }, 8219 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 8220 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 8221 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 8222 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 8223 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 8224 }; 8225 CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 8226 8227 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 8228 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8229 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 8230 { "Single Colls", KSTAT_KV_U_PACKETS, }, 8231 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 8232 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 8233 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 8234 { "Late Colls", KSTAT_KV_U_NONE, }, 8235 { "Exess Colls", KSTAT_KV_U_NONE, }, 8236 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 8237 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 8238 { "Too Long", KSTAT_KV_U_PACKETS, }, 8239 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 8240 { "Symbol Errs", KSTAT_KV_U_NONE, }, 8241 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 8242 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8243 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8244 }; 8245 CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 8246 8247 struct mcx_kstat_ppcnt { 8248 const char *ksp_name; 8249 const struct mcx_ppcnt *ksp_tpl; 8250 unsigned int ksp_n; 8251 uint8_t ksp_grp; 8252 }; 8253 8254 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 8255 .ksp_name = "ieee802.3", 8256 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 8257 .ksp_n = nitems(mcx_ppcnt_ieee8023_tpl), 8258 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 8259 }; 8260 8261 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 8262 .ksp_name = "rfc2863", 8263 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 8264 .ksp_n = nitems(mcx_ppcnt_rfc2863_tpl), 8265 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 8266 }; 8267 8268 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 8269 .ksp_name = "rfc2819", 8270 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 8271 .ksp_n = nitems(mcx_ppcnt_rfc2819_tpl), 8272 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 8273 }; 8274 8275 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 8276 .ksp_name = "rfc3635", 8277 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 8278 .ksp_n = nitems(mcx_ppcnt_rfc3635_tpl), 8279 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 8280 }; 8281 8282 static int mcx_kstat_ppcnt_read(struct kstat *); 8283 8284 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 8285 static void mcx_kstat_attach_queues(struct mcx_softc *sc); 8286 8287 static struct kstat * 8288 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 8289 const struct mcx_kstat_ppcnt *ksp) 8290 { 8291 struct kstat *ks; 8292 struct kstat_kv *kvs; 8293 unsigned int i; 8294 8295 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 8296 if (ks == NULL) 8297 return (NULL); 8298 8299 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 8300 M_DEVBUF, M_WAITOK); 8301 8302 for (i = 0; i < ksp->ksp_n; i++) { 8303 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 8304 8305 kstat_kv_unit_init(&kvs[i], tpl->name, 8306 KSTAT_KV_T_COUNTER64, tpl->unit); 8307 } 8308 8309 ks->ks_softc = sc; 8310 ks->ks_ptr = (void *)ksp; 8311 ks->ks_data = kvs; 8312 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 8313 ks->ks_read = mcx_kstat_ppcnt_read; 8314 8315 kstat_install(ks); 8316 8317 return (ks); 8318 } 8319 8320 static void 8321 mcx_kstat_attach(struct mcx_softc *sc) 8322 { 8323 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 8324 &mcx_kstat_ppcnt_ieee8023); 8325 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 8326 &mcx_kstat_ppcnt_rfc2863); 8327 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 8328 &mcx_kstat_ppcnt_rfc2819); 8329 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 8330 &mcx_kstat_ppcnt_rfc3635); 8331 8332 mcx_kstat_attach_tmps(sc); 8333 mcx_kstat_attach_queues(sc); 8334 } 8335 8336 static int 8337 mcx_kstat_ppcnt_read(struct kstat *ks) 8338 { 8339 struct mcx_softc *sc = ks->ks_softc; 8340 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 8341 struct mcx_reg_ppcnt ppcnt = { 8342 .ppcnt_grp = ksp->ksp_grp, 8343 .ppcnt_local_port = 1, 8344 }; 8345 struct kstat_kv *kvs = ks->ks_data; 8346 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 8347 unsigned int i; 8348 int rv; 8349 8350 KERNEL_LOCK(); /* XXX */ 8351 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 8352 &ppcnt, sizeof(ppcnt)); 8353 KERNEL_UNLOCK(); 8354 if (rv != 0) 8355 return (EIO); 8356 8357 nanouptime(&ks->ks_updated); 8358 8359 for (i = 0; i < ksp->ksp_n; i++) 8360 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 8361 8362 return (0); 8363 } 8364 8365 struct mcx_kstat_mtmp { 8366 struct kstat_kv ktmp_name; 8367 struct kstat_kv ktmp_temperature; 8368 struct kstat_kv ktmp_threshold_lo; 8369 struct kstat_kv ktmp_threshold_hi; 8370 }; 8371 8372 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 8373 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 8374 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 8375 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 8376 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 8377 }; 8378 8379 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 8380 8381 static int mcx_kstat_mtmp_read(struct kstat *); 8382 8383 static void 8384 mcx_kstat_attach_tmps(struct mcx_softc *sc) 8385 { 8386 struct kstat *ks; 8387 struct mcx_reg_mtcap mtcap; 8388 struct mcx_kstat_mtmp *ktmp; 8389 uint64_t map; 8390 unsigned int i, n; 8391 8392 memset(&mtcap, 0, sizeof(mtcap)); 8393 8394 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8395 &mtcap, sizeof(mtcap)) != 0) { 8396 /* unable to find temperature sensors */ 8397 return; 8398 } 8399 8400 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8401 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8402 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8403 8404 n = 0; 8405 map = bemtoh64(&mtcap.mtcap_sensor_map); 8406 for (i = 0; i < sizeof(map) * NBBY; i++) { 8407 if (!ISSET(map, (1ULL << i))) 8408 continue; 8409 8410 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8411 KSTAT_T_KV, 0); 8412 if (ks == NULL) { 8413 /* unable to attach temperature sensor %u, i */ 8414 continue; 8415 } 8416 8417 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8418 *ktmp = mcx_kstat_mtmp_tpl; 8419 8420 ks->ks_data = ktmp; 8421 ks->ks_datalen = sizeof(*ktmp); 8422 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8423 ks->ks_read = mcx_kstat_mtmp_read; 8424 8425 ks->ks_softc = sc; 8426 kstat_install(ks); 8427 8428 sc->sc_kstat_mtmp[n++] = ks; 8429 if (n >= sc->sc_kstat_mtmp_count) 8430 break; 8431 } 8432 } 8433 8434 static uint64_t 8435 mcx_tmp_to_uK(uint16_t *t) 8436 { 8437 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8438 mt *= 1000000 / 8; /* convert to uC */ 8439 mt += 273150000; /* convert to uK */ 8440 8441 return (mt); 8442 } 8443 8444 static int 8445 mcx_kstat_mtmp_read(struct kstat *ks) 8446 { 8447 struct mcx_softc *sc = ks->ks_softc; 8448 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8449 struct mcx_reg_mtmp mtmp; 8450 int rv; 8451 struct timeval updated; 8452 8453 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8454 8455 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8456 return (0); 8457 8458 memset(&mtmp, 0, sizeof(mtmp)); 8459 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8460 8461 KERNEL_LOCK(); /* XXX */ 8462 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8463 &mtmp, sizeof(mtmp)); 8464 KERNEL_UNLOCK(); 8465 if (rv != 0) 8466 return (EIO); 8467 8468 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8469 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8470 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8471 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8472 kstat_kv_temp(&ktmp->ktmp_temperature) = 8473 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8474 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8475 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8476 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8477 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8478 8479 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8480 8481 return (0); 8482 } 8483 8484 struct mcx_queuestat { 8485 char name[KSTAT_KV_NAMELEN]; 8486 enum kstat_kv_type type; 8487 }; 8488 8489 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = { 8490 { "RQ SW prod", KSTAT_KV_T_COUNTER64 }, 8491 { "RQ HW prod", KSTAT_KV_T_COUNTER64 }, 8492 { "RQ HW cons", KSTAT_KV_T_COUNTER64 }, 8493 { "RQ HW state", KSTAT_KV_T_ISTR }, 8494 8495 { "SQ SW prod", KSTAT_KV_T_COUNTER64 }, 8496 { "SQ SW cons", KSTAT_KV_T_COUNTER64 }, 8497 { "SQ HW prod", KSTAT_KV_T_COUNTER64 }, 8498 { "SQ HW cons", KSTAT_KV_T_COUNTER64 }, 8499 { "SQ HW state", KSTAT_KV_T_ISTR }, 8500 8501 { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, 8502 { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, 8503 { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, 8504 { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, 8505 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, 8506 { "CQ HW status", KSTAT_KV_T_ISTR }, 8507 { "CQ HW state", KSTAT_KV_T_ISTR }, 8508 8509 { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, 8510 { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, 8511 { "EQ HW cons", KSTAT_KV_T_COUNTER64 }, 8512 { "EQ HW status", KSTAT_KV_T_ISTR }, 8513 { "EQ HW state", KSTAT_KV_T_ISTR }, 8514 }; 8515 8516 static int mcx_kstat_queue_read(struct kstat *); 8517 8518 static void 8519 mcx_kstat_attach_queues(struct mcx_softc *sc) 8520 { 8521 struct kstat *ks; 8522 struct kstat_kv *kvs; 8523 int q, i; 8524 8525 for (q = 0; q < sc->sc_nqueues; q++) { 8526 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, 8527 KSTAT_T_KV, 0); 8528 if (ks == NULL) { 8529 /* unable to attach queue stats %u, q */ 8530 continue; 8531 } 8532 8533 kvs = mallocarray(nitems(mcx_queue_kstat_tpl), 8534 sizeof(*kvs), M_DEVBUF, M_WAITOK); 8535 8536 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) { 8537 const struct mcx_queuestat *tpl = 8538 &mcx_queue_kstat_tpl[i]; 8539 8540 kstat_kv_init(&kvs[i], tpl->name, tpl->type); 8541 } 8542 8543 ks->ks_softc = &sc->sc_queues[q]; 8544 ks->ks_data = kvs; 8545 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs); 8546 ks->ks_read = mcx_kstat_queue_read; 8547 8548 sc->sc_queues[q].q_kstat = ks; 8549 kstat_install(ks); 8550 } 8551 } 8552 8553 static int 8554 mcx_kstat_queue_read(struct kstat *ks) 8555 { 8556 struct mcx_queues *q = ks->ks_softc; 8557 struct mcx_softc *sc = q->q_sc; 8558 struct kstat_kv *kvs = ks->ks_data; 8559 union { 8560 struct mcx_rq_ctx rq; 8561 struct mcx_sq_ctx sq; 8562 struct mcx_cq_ctx cq; 8563 struct mcx_eq_ctx eq; 8564 } u; 8565 const char *text; 8566 int error = 0; 8567 8568 KERNEL_LOCK(); 8569 8570 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) { 8571 error = EIO; 8572 goto out; 8573 } 8574 8575 kstat_kv_u64(kvs++) = q->q_rx.rx_prod; 8576 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter); 8577 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter); 8578 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >> 8579 MCX_RQ_CTX_STATE_SHIFT) { 8580 case MCX_RQ_CTX_STATE_RST: 8581 text = "RST"; 8582 break; 8583 case MCX_RQ_CTX_STATE_RDY: 8584 text = "RDY"; 8585 break; 8586 case MCX_RQ_CTX_STATE_ERR: 8587 text = "ERR"; 8588 break; 8589 default: 8590 text = "unknown"; 8591 break; 8592 } 8593 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8594 kvs++; 8595 8596 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) { 8597 error = EIO; 8598 goto out; 8599 } 8600 8601 kstat_kv_u64(kvs++) = q->q_tx.tx_prod; 8602 kstat_kv_u64(kvs++) = q->q_tx.tx_cons; 8603 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter); 8604 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter); 8605 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >> 8606 MCX_SQ_CTX_STATE_SHIFT) { 8607 case MCX_SQ_CTX_STATE_RST: 8608 text = "RST"; 8609 break; 8610 case MCX_SQ_CTX_STATE_RDY: 8611 text = "RDY"; 8612 break; 8613 case MCX_SQ_CTX_STATE_ERR: 8614 text = "ERR"; 8615 break; 8616 default: 8617 text = "unknown"; 8618 break; 8619 } 8620 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8621 kvs++; 8622 8623 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) { 8624 error = EIO; 8625 goto out; 8626 } 8627 8628 kstat_kv_u64(kvs++) = q->q_cq.cq_cons; 8629 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); 8630 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); 8631 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); 8632 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); 8633 8634 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >> 8635 MCX_CQ_CTX_STATUS_SHIFT) { 8636 case MCX_CQ_CTX_STATUS_OK: 8637 text = "OK"; 8638 break; 8639 case MCX_CQ_CTX_STATUS_OVERFLOW: 8640 text = "overflow"; 8641 break; 8642 case MCX_CQ_CTX_STATUS_WRITE_FAIL: 8643 text = "write fail"; 8644 break; 8645 default: 8646 text = "unknown"; 8647 break; 8648 } 8649 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8650 kvs++; 8651 8652 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >> 8653 MCX_CQ_CTX_STATE_SHIFT) { 8654 case MCX_CQ_CTX_STATE_SOLICITED: 8655 text = "solicited"; 8656 break; 8657 case MCX_CQ_CTX_STATE_ARMED: 8658 text = "armed"; 8659 break; 8660 case MCX_CQ_CTX_STATE_FIRED: 8661 text = "fired"; 8662 break; 8663 default: 8664 text = "unknown"; 8665 break; 8666 } 8667 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8668 kvs++; 8669 8670 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { 8671 error = EIO; 8672 goto out; 8673 } 8674 8675 kstat_kv_u64(kvs++) = q->q_eq.eq_cons; 8676 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter); 8677 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter); 8678 8679 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >> 8680 MCX_EQ_CTX_STATUS_SHIFT) { 8681 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE: 8682 text = "write fail"; 8683 break; 8684 case MCX_EQ_CTX_STATUS_OK: 8685 text = "OK"; 8686 break; 8687 default: 8688 text = "unknown"; 8689 break; 8690 } 8691 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8692 kvs++; 8693 8694 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >> 8695 MCX_EQ_CTX_STATE_SHIFT) { 8696 case MCX_EQ_CTX_STATE_ARMED: 8697 text = "armed"; 8698 break; 8699 case MCX_EQ_CTX_STATE_FIRED: 8700 text = "fired"; 8701 break; 8702 default: 8703 text = "unknown"; 8704 break; 8705 } 8706 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8707 kvs++; 8708 8709 nanouptime(&ks->ks_updated); 8710 out: 8711 KERNEL_UNLOCK(); 8712 return (error); 8713 } 8714 8715 #endif /* NKSTAT > 0 */ 8716 8717 static unsigned int 8718 mcx_timecounter_read(struct timecounter *tc) 8719 { 8720 struct mcx_softc *sc = tc->tc_priv; 8721 8722 return (mcx_rd(sc, MCX_INTERNAL_TIMER_L)); 8723 } 8724 8725 static void 8726 mcx_timecounter_attach(struct mcx_softc *sc) 8727 { 8728 struct timecounter *tc = &sc->sc_timecounter; 8729 8730 tc->tc_get_timecount = mcx_timecounter_read; 8731 tc->tc_counter_mask = ~0U; 8732 tc->tc_frequency = sc->sc_khz * 1000; 8733 tc->tc_name = sc->sc_dev.dv_xname; 8734 tc->tc_quality = -100; 8735 tc->tc_priv = sc; 8736 8737 tc_init(tc); 8738 } 8739