1 /* $OpenBSD: if_mcx.c,v 1.91 2020/12/27 01:00:25 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 #include "vlan.h" 22 #include "kstat.h" 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/kernel.h> 29 #include <sys/socket.h> 30 #include <sys/device.h> 31 #include <sys/pool.h> 32 #include <sys/queue.h> 33 #include <sys/timeout.h> 34 #include <sys/task.h> 35 #include <sys/atomic.h> 36 #include <sys/timetc.h> 37 38 #include <machine/bus.h> 39 #include <machine/intr.h> 40 41 #include <net/if.h> 42 #include <net/if_dl.h> 43 #include <net/if_media.h> 44 #include <net/toeplitz.h> 45 46 #if NBPFILTER > 0 47 #include <net/bpf.h> 48 #endif 49 50 #if NKSTAT > 0 51 #include <sys/kstat.h> 52 #endif 53 54 #include <netinet/in.h> 55 #include <netinet/if_ether.h> 56 57 #include <dev/pci/pcireg.h> 58 #include <dev/pci/pcivar.h> 59 #include <dev/pci/pcidevs.h> 60 61 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 62 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 63 64 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 65 66 #define MCX_FW_VER 0x0000 67 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 68 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 69 #define MCX_CMDIF_FW_SUBVER 0x0004 70 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 71 #define MCX_CMDIF(_v) ((_v) >> 16) 72 73 #define MCX_ISSI 1 /* as per the PRM */ 74 #define MCX_CMD_IF_SUPPORTED 5 75 76 #define MCX_HARDMTU 9500 77 78 #define MCX_PAGE_SHIFT 12 79 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 80 81 /* queue sizes */ 82 #define MCX_LOG_EQ_SIZE 7 83 #define MCX_LOG_CQ_SIZE 12 84 #define MCX_LOG_RQ_SIZE 10 85 #define MCX_LOG_SQ_SIZE 11 86 87 #define MCX_MAX_QUEUES 1 88 89 /* completion event moderation - about 10khz, or 90% of the cq */ 90 #define MCX_CQ_MOD_PERIOD 50 91 #define MCX_CQ_MOD_COUNTER \ 92 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 93 94 #define MCX_LOG_SQ_ENTRY_SIZE 6 95 #define MCX_SQ_ENTRY_MAX_SLOTS 4 96 #define MCX_SQ_SEGS_PER_SLOT \ 97 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 98 #define MCX_SQ_MAX_SEGMENTS \ 99 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 100 101 #define MCX_LOG_FLOW_TABLE_SIZE 5 102 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 103 #define MCX_NUM_MCAST_FLOWS \ 104 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 105 106 #define MCX_SQ_INLINE_SIZE 18 107 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 108 109 /* doorbell offsets */ 110 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 111 112 #define MCX_CQ_DOORBELL_BASE 0 113 #define MCX_CQ_DOORBELL_STRIDE 64 114 115 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 116 #define MCX_WQ_DOORBELL_STRIDE 64 117 /* make sure the doorbells fit */ 118 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 119 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 120 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 121 122 #define MCX_WQ_DOORBELL_MASK 0xffff 123 124 /* uar registers */ 125 #define MCX_UAR_CQ_DOORBELL 0x20 126 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 127 #define MCX_UAR_EQ_DOORBELL 0x48 128 #define MCX_UAR_BF 0x800 129 130 #define MCX_CMDQ_ADDR_HI 0x0010 131 #define MCX_CMDQ_ADDR_LO 0x0014 132 #define MCX_CMDQ_ADDR_NMASK 0xfff 133 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 134 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 135 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 136 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 137 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 138 139 #define MCX_CMDQ_DOORBELL 0x0018 140 141 #define MCX_STATE 0x01fc 142 #define MCX_STATE_MASK (1 << 31) 143 #define MCX_STATE_INITIALIZING (1 << 31) 144 #define MCX_STATE_READY (0 << 31) 145 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 146 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 147 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 148 149 #define MCX_INTERNAL_TIMER 0x1000 150 #define MCX_INTERNAL_TIMER_H 0x1000 151 #define MCX_INTERNAL_TIMER_L 0x1004 152 153 #define MCX_CLEAR_INT 0x100c 154 155 #define MCX_REG_OP_WRITE 0 156 #define MCX_REG_OP_READ 1 157 158 #define MCX_REG_PMLP 0x5002 159 #define MCX_REG_PMTU 0x5003 160 #define MCX_REG_PTYS 0x5004 161 #define MCX_REG_PAOS 0x5006 162 #define MCX_REG_PFCC 0x5007 163 #define MCX_REG_PPCNT 0x5008 164 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 165 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 166 #define MCX_REG_MCIA 0x9014 167 168 #define MCX_ETHER_CAP_SGMII 0 169 #define MCX_ETHER_CAP_1000_KX 1 170 #define MCX_ETHER_CAP_10G_CX4 2 171 #define MCX_ETHER_CAP_10G_KX4 3 172 #define MCX_ETHER_CAP_10G_KR 4 173 #define MCX_ETHER_CAP_40G_CR4 6 174 #define MCX_ETHER_CAP_40G_KR4 7 175 #define MCX_ETHER_CAP_10G_CR 12 176 #define MCX_ETHER_CAP_10G_SR 13 177 #define MCX_ETHER_CAP_10G_LR 14 178 #define MCX_ETHER_CAP_40G_SR4 15 179 #define MCX_ETHER_CAP_40G_LR4 16 180 #define MCX_ETHER_CAP_50G_SR2 18 181 #define MCX_ETHER_CAP_100G_CR4 20 182 #define MCX_ETHER_CAP_100G_SR4 21 183 #define MCX_ETHER_CAP_100G_KR4 22 184 #define MCX_ETHER_CAP_25G_CR 27 185 #define MCX_ETHER_CAP_25G_KR 28 186 #define MCX_ETHER_CAP_25G_SR 29 187 #define MCX_ETHER_CAP_50G_CR2 30 188 #define MCX_ETHER_CAP_50G_KR2 31 189 190 #define MCX_MAX_CQE 32 191 192 #define MCX_CMD_QUERY_HCA_CAP 0x100 193 #define MCX_CMD_QUERY_ADAPTER 0x101 194 #define MCX_CMD_INIT_HCA 0x102 195 #define MCX_CMD_TEARDOWN_HCA 0x103 196 #define MCX_CMD_ENABLE_HCA 0x104 197 #define MCX_CMD_DISABLE_HCA 0x105 198 #define MCX_CMD_QUERY_PAGES 0x107 199 #define MCX_CMD_MANAGE_PAGES 0x108 200 #define MCX_CMD_SET_HCA_CAP 0x109 201 #define MCX_CMD_QUERY_ISSI 0x10a 202 #define MCX_CMD_SET_ISSI 0x10b 203 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 204 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 205 #define MCX_CMD_CREATE_EQ 0x301 206 #define MCX_CMD_DESTROY_EQ 0x302 207 #define MCX_CMD_QUERY_EQ 0x303 208 #define MCX_CMD_CREATE_CQ 0x400 209 #define MCX_CMD_DESTROY_CQ 0x401 210 #define MCX_CMD_QUERY_CQ 0x402 211 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 212 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 213 0x755 214 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 215 #define MCX_CMD_ALLOC_PD 0x800 216 #define MCX_CMD_ALLOC_UAR 0x802 217 #define MCX_CMD_ACCESS_REG 0x805 218 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 219 #define MCX_CMD_CREATE_TIR 0x900 220 #define MCX_CMD_DESTROY_TIR 0x902 221 #define MCX_CMD_CREATE_SQ 0x904 222 #define MCX_CMD_MODIFY_SQ 0x905 223 #define MCX_CMD_DESTROY_SQ 0x906 224 #define MCX_CMD_QUERY_SQ 0x907 225 #define MCX_CMD_CREATE_RQ 0x908 226 #define MCX_CMD_MODIFY_RQ 0x909 227 #define MCX_CMD_DESTROY_RQ 0x90a 228 #define MCX_CMD_QUERY_RQ 0x90b 229 #define MCX_CMD_CREATE_TIS 0x912 230 #define MCX_CMD_DESTROY_TIS 0x914 231 #define MCX_CMD_CREATE_RQT 0x916 232 #define MCX_CMD_DESTROY_RQT 0x918 233 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 234 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 235 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 236 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 237 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 238 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 239 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 240 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 241 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 242 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 243 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 244 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 245 246 #define MCX_QUEUE_STATE_RST 0 247 #define MCX_QUEUE_STATE_RDY 1 248 #define MCX_QUEUE_STATE_ERR 3 249 250 #define MCX_FLOW_TABLE_TYPE_RX 0 251 #define MCX_FLOW_TABLE_TYPE_TX 1 252 253 #define MCX_CMDQ_INLINE_DATASIZE 16 254 255 struct mcx_cmdq_entry { 256 uint8_t cq_type; 257 #define MCX_CMDQ_TYPE_PCIE 0x7 258 uint8_t cq_reserved0[3]; 259 260 uint32_t cq_input_length; 261 uint64_t cq_input_ptr; 262 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 263 264 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 265 uint64_t cq_output_ptr; 266 uint32_t cq_output_length; 267 268 uint8_t cq_token; 269 uint8_t cq_signature; 270 uint8_t cq_reserved1[1]; 271 uint8_t cq_status; 272 #define MCX_CQ_STATUS_SHIFT 1 273 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 274 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 275 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 276 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 277 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 278 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 279 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 280 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 281 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 282 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 283 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 284 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 285 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 286 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 287 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 288 (0x10 << MCX_CQ_STATUS_SHIFT) 289 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 290 #define MCX_CQ_STATUS_OWN_MASK 0x1 291 #define MCX_CQ_STATUS_OWN_SW 0x0 292 #define MCX_CQ_STATUS_OWN_HW 0x1 293 } __packed __aligned(8); 294 295 #define MCX_CMDQ_MAILBOX_DATASIZE 512 296 297 struct mcx_cmdq_mailbox { 298 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 299 uint8_t mb_reserved0[48]; 300 uint64_t mb_next_ptr; 301 uint32_t mb_block_number; 302 uint8_t mb_reserved1[1]; 303 uint8_t mb_token; 304 uint8_t mb_ctrl_signature; 305 uint8_t mb_signature; 306 } __packed __aligned(8); 307 308 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 309 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 310 MCX_CMDQ_MAILBOX_ALIGN) 311 /* 312 * command mailbox structres 313 */ 314 315 struct mcx_cmd_enable_hca_in { 316 uint16_t cmd_opcode; 317 uint8_t cmd_reserved0[4]; 318 uint16_t cmd_op_mod; 319 uint8_t cmd_reserved1[2]; 320 uint16_t cmd_function_id; 321 uint8_t cmd_reserved2[4]; 322 } __packed __aligned(4); 323 324 struct mcx_cmd_enable_hca_out { 325 uint8_t cmd_status; 326 uint8_t cmd_reserved0[3]; 327 uint32_t cmd_syndrome; 328 uint8_t cmd_reserved1[4]; 329 } __packed __aligned(4); 330 331 struct mcx_cmd_init_hca_in { 332 uint16_t cmd_opcode; 333 uint8_t cmd_reserved0[4]; 334 uint16_t cmd_op_mod; 335 uint8_t cmd_reserved1[8]; 336 } __packed __aligned(4); 337 338 struct mcx_cmd_init_hca_out { 339 uint8_t cmd_status; 340 uint8_t cmd_reserved0[3]; 341 uint32_t cmd_syndrome; 342 uint8_t cmd_reserved1[8]; 343 } __packed __aligned(4); 344 345 struct mcx_cmd_teardown_hca_in { 346 uint16_t cmd_opcode; 347 uint8_t cmd_reserved0[4]; 348 uint16_t cmd_op_mod; 349 uint8_t cmd_reserved1[2]; 350 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 351 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 352 uint16_t cmd_profile; 353 uint8_t cmd_reserved2[4]; 354 } __packed __aligned(4); 355 356 struct mcx_cmd_teardown_hca_out { 357 uint8_t cmd_status; 358 uint8_t cmd_reserved0[3]; 359 uint32_t cmd_syndrome; 360 uint8_t cmd_reserved1[8]; 361 } __packed __aligned(4); 362 363 struct mcx_cmd_access_reg_in { 364 uint16_t cmd_opcode; 365 uint8_t cmd_reserved0[4]; 366 uint16_t cmd_op_mod; 367 uint8_t cmd_reserved1[2]; 368 uint16_t cmd_register_id; 369 uint32_t cmd_argument; 370 } __packed __aligned(4); 371 372 struct mcx_cmd_access_reg_out { 373 uint8_t cmd_status; 374 uint8_t cmd_reserved0[3]; 375 uint32_t cmd_syndrome; 376 uint8_t cmd_reserved1[8]; 377 } __packed __aligned(4); 378 379 struct mcx_reg_pmtu { 380 uint8_t rp_reserved1; 381 uint8_t rp_local_port; 382 uint8_t rp_reserved2[2]; 383 uint16_t rp_max_mtu; 384 uint8_t rp_reserved3[2]; 385 uint16_t rp_admin_mtu; 386 uint8_t rp_reserved4[2]; 387 uint16_t rp_oper_mtu; 388 uint8_t rp_reserved5[2]; 389 } __packed __aligned(4); 390 391 struct mcx_reg_ptys { 392 uint8_t rp_reserved1; 393 uint8_t rp_local_port; 394 uint8_t rp_reserved2; 395 uint8_t rp_proto_mask; 396 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 397 uint8_t rp_reserved3[8]; 398 uint32_t rp_eth_proto_cap; 399 uint8_t rp_reserved4[8]; 400 uint32_t rp_eth_proto_admin; 401 uint8_t rp_reserved5[8]; 402 uint32_t rp_eth_proto_oper; 403 uint8_t rp_reserved6[24]; 404 } __packed __aligned(4); 405 406 struct mcx_reg_paos { 407 uint8_t rp_reserved1; 408 uint8_t rp_local_port; 409 uint8_t rp_admin_status; 410 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 411 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 412 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 413 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 414 uint8_t rp_oper_status; 415 #define MCX_REG_PAOS_OPER_STATUS_UP 1 416 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 417 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 418 uint8_t rp_admin_state_update; 419 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 420 uint8_t rp_reserved2[11]; 421 } __packed __aligned(4); 422 423 struct mcx_reg_pfcc { 424 uint8_t rp_reserved1; 425 uint8_t rp_local_port; 426 uint8_t rp_reserved2[3]; 427 uint8_t rp_prio_mask_tx; 428 uint8_t rp_reserved3; 429 uint8_t rp_prio_mask_rx; 430 uint8_t rp_pptx_aptx; 431 uint8_t rp_pfctx; 432 uint8_t rp_fctx_dis; 433 uint8_t rp_reserved4; 434 uint8_t rp_pprx_aprx; 435 uint8_t rp_pfcrx; 436 uint8_t rp_reserved5[2]; 437 uint16_t rp_dev_stall_min; 438 uint16_t rp_dev_stall_crit; 439 uint8_t rp_reserved6[12]; 440 } __packed __aligned(4); 441 442 #define MCX_PMLP_MODULE_NUM_MASK 0xff 443 struct mcx_reg_pmlp { 444 uint8_t rp_rxtx; 445 uint8_t rp_local_port; 446 uint8_t rp_reserved0; 447 uint8_t rp_width; 448 uint32_t rp_lane0_mapping; 449 uint32_t rp_lane1_mapping; 450 uint32_t rp_lane2_mapping; 451 uint32_t rp_lane3_mapping; 452 uint8_t rp_reserved1[44]; 453 } __packed __aligned(4); 454 455 struct mcx_reg_ppcnt { 456 uint8_t ppcnt_swid; 457 uint8_t ppcnt_local_port; 458 uint8_t ppcnt_pnat; 459 uint8_t ppcnt_grp; 460 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 461 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 462 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 463 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 464 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 465 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 466 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 467 468 uint8_t ppcnt_clr; 469 uint8_t ppcnt_reserved1[2]; 470 uint8_t ppcnt_prio_tc; 471 #define MCX_REG_PPCNT_CLR (1 << 7) 472 473 uint8_t ppcnt_counter_set[248]; 474 } __packed __aligned(8); 475 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 476 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 477 sizeof(uint64_t)) == 0); 478 479 enum mcx_ppcnt_ieee8023 { 480 frames_transmitted_ok, 481 frames_received_ok, 482 frame_check_sequence_errors, 483 alignment_errors, 484 octets_transmitted_ok, 485 octets_received_ok, 486 multicast_frames_xmitted_ok, 487 broadcast_frames_xmitted_ok, 488 multicast_frames_received_ok, 489 broadcast_frames_received_ok, 490 in_range_length_errors, 491 out_of_range_length_field, 492 frame_too_long_errors, 493 symbol_error_during_carrier, 494 mac_control_frames_transmitted, 495 mac_control_frames_received, 496 unsupported_opcodes_received, 497 pause_mac_ctrl_frames_received, 498 pause_mac_ctrl_frames_transmitted, 499 500 mcx_ppcnt_ieee8023_count 501 }; 502 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 503 504 enum mcx_ppcnt_rfc2863 { 505 in_octets, 506 in_ucast_pkts, 507 in_discards, 508 in_errors, 509 in_unknown_protos, 510 out_octets, 511 out_ucast_pkts, 512 out_discards, 513 out_errors, 514 in_multicast_pkts, 515 in_broadcast_pkts, 516 out_multicast_pkts, 517 out_broadcast_pkts, 518 519 mcx_ppcnt_rfc2863_count 520 }; 521 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 522 523 enum mcx_ppcnt_rfc2819 { 524 drop_events, 525 octets, 526 pkts, 527 broadcast_pkts, 528 multicast_pkts, 529 crc_align_errors, 530 undersize_pkts, 531 oversize_pkts, 532 fragments, 533 jabbers, 534 collisions, 535 pkts64octets, 536 pkts65to127octets, 537 pkts128to255octets, 538 pkts256to511octets, 539 pkts512to1023octets, 540 pkts1024to1518octets, 541 pkts1519to2047octets, 542 pkts2048to4095octets, 543 pkts4096to8191octets, 544 pkts8192to10239octets, 545 546 mcx_ppcnt_rfc2819_count 547 }; 548 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 549 550 enum mcx_ppcnt_rfc3635 { 551 dot3stats_alignment_errors, 552 dot3stats_fcs_errors, 553 dot3stats_single_collision_frames, 554 dot3stats_multiple_collision_frames, 555 dot3stats_sqe_test_errors, 556 dot3stats_deferred_transmissions, 557 dot3stats_late_collisions, 558 dot3stats_excessive_collisions, 559 dot3stats_internal_mac_transmit_errors, 560 dot3stats_carrier_sense_errors, 561 dot3stats_frame_too_longs, 562 dot3stats_internal_mac_receive_errors, 563 dot3stats_symbol_errors, 564 dot3control_in_unknown_opcodes, 565 dot3in_pause_frames, 566 dot3out_pause_frames, 567 568 mcx_ppcnt_rfc3635_count 569 }; 570 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 571 572 struct mcx_reg_mtcap { 573 uint8_t _reserved1[3]; 574 uint8_t mtcap_sensor_count; 575 uint8_t _reserved2[4]; 576 577 uint64_t mtcap_sensor_map; 578 }; 579 580 struct mcx_reg_mtmp { 581 uint8_t _reserved1[2]; 582 uint16_t mtmp_sensor_index; 583 584 uint8_t _reserved2[2]; 585 uint16_t mtmp_temperature; 586 587 uint16_t mtmp_mte_mtr; 588 #define MCX_REG_MTMP_MTE (1 << 15) 589 #define MCX_REG_MTMP_MTR (1 << 14) 590 uint16_t mtmp_max_temperature; 591 592 uint16_t mtmp_tee; 593 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 594 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 595 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 596 uint16_t mtmp_temperature_threshold_hi; 597 598 uint8_t _reserved3[2]; 599 uint16_t mtmp_temperature_threshold_lo; 600 601 uint8_t _reserved4[4]; 602 603 uint8_t mtmp_sensor_name[8]; 604 }; 605 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 606 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 607 608 #define MCX_MCIA_EEPROM_BYTES 32 609 struct mcx_reg_mcia { 610 uint8_t rm_l; 611 uint8_t rm_module; 612 uint8_t rm_reserved0; 613 uint8_t rm_status; 614 uint8_t rm_i2c_addr; 615 uint8_t rm_page_num; 616 uint16_t rm_dev_addr; 617 uint16_t rm_reserved1; 618 uint16_t rm_size; 619 uint32_t rm_reserved2; 620 uint8_t rm_data[48]; 621 } __packed __aligned(4); 622 623 struct mcx_cmd_query_issi_in { 624 uint16_t cmd_opcode; 625 uint8_t cmd_reserved0[4]; 626 uint16_t cmd_op_mod; 627 uint8_t cmd_reserved1[8]; 628 } __packed __aligned(4); 629 630 struct mcx_cmd_query_issi_il_out { 631 uint8_t cmd_status; 632 uint8_t cmd_reserved0[3]; 633 uint32_t cmd_syndrome; 634 uint8_t cmd_reserved1[2]; 635 uint16_t cmd_current_issi; 636 uint8_t cmd_reserved2[4]; 637 } __packed __aligned(4); 638 639 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 640 641 struct mcx_cmd_query_issi_mb_out { 642 uint8_t cmd_reserved2[16]; 643 uint8_t cmd_supported_issi[80]; /* very big endian */ 644 } __packed __aligned(4); 645 646 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 647 648 struct mcx_cmd_set_issi_in { 649 uint16_t cmd_opcode; 650 uint8_t cmd_reserved0[4]; 651 uint16_t cmd_op_mod; 652 uint8_t cmd_reserved1[2]; 653 uint16_t cmd_current_issi; 654 uint8_t cmd_reserved2[4]; 655 } __packed __aligned(4); 656 657 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 658 659 struct mcx_cmd_set_issi_out { 660 uint8_t cmd_status; 661 uint8_t cmd_reserved0[3]; 662 uint32_t cmd_syndrome; 663 uint8_t cmd_reserved1[8]; 664 } __packed __aligned(4); 665 666 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 667 668 struct mcx_cmd_query_pages_in { 669 uint16_t cmd_opcode; 670 uint8_t cmd_reserved0[4]; 671 uint16_t cmd_op_mod; 672 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 673 #define MCX_CMD_QUERY_PAGES_INIT 0x02 674 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 675 uint8_t cmd_reserved1[8]; 676 } __packed __aligned(4); 677 678 struct mcx_cmd_query_pages_out { 679 uint8_t cmd_status; 680 uint8_t cmd_reserved0[3]; 681 uint32_t cmd_syndrome; 682 uint8_t cmd_reserved1[2]; 683 uint16_t cmd_func_id; 684 int32_t cmd_num_pages; 685 } __packed __aligned(4); 686 687 struct mcx_cmd_manage_pages_in { 688 uint16_t cmd_opcode; 689 uint8_t cmd_reserved0[4]; 690 uint16_t cmd_op_mod; 691 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 692 0x00 693 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 694 0x01 695 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 696 0x02 697 uint8_t cmd_reserved1[2]; 698 uint16_t cmd_func_id; 699 uint32_t cmd_input_num_entries; 700 } __packed __aligned(4); 701 702 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 703 704 struct mcx_cmd_manage_pages_out { 705 uint8_t cmd_status; 706 uint8_t cmd_reserved0[3]; 707 uint32_t cmd_syndrome; 708 uint32_t cmd_output_num_entries; 709 uint8_t cmd_reserved1[4]; 710 } __packed __aligned(4); 711 712 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 713 714 struct mcx_cmd_query_hca_cap_in { 715 uint16_t cmd_opcode; 716 uint8_t cmd_reserved0[4]; 717 uint16_t cmd_op_mod; 718 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 719 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 720 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 721 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 722 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 723 uint8_t cmd_reserved1[8]; 724 } __packed __aligned(4); 725 726 struct mcx_cmd_query_hca_cap_out { 727 uint8_t cmd_status; 728 uint8_t cmd_reserved0[3]; 729 uint32_t cmd_syndrome; 730 uint8_t cmd_reserved1[8]; 731 } __packed __aligned(4); 732 733 #define MCX_HCA_CAP_LEN 0x1000 734 #define MCX_HCA_CAP_NMAILBOXES \ 735 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 736 737 #if __GNUC_PREREQ__(4, 3) 738 #define __counter__ __COUNTER__ 739 #else 740 #define __counter__ __LINE__ 741 #endif 742 743 #define __token(_tok, _num) _tok##_num 744 #define _token(_tok, _num) __token(_tok, _num) 745 #define __reserved__ _token(__reserved, __counter__) 746 747 struct mcx_cap_device { 748 uint8_t reserved0[16]; 749 750 uint8_t log_max_srq_sz; 751 uint8_t log_max_qp_sz; 752 uint8_t __reserved__[1]; 753 uint8_t log_max_qp; /* 5 bits */ 754 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 755 756 uint8_t __reserved__[1]; 757 uint8_t log_max_srq; /* 5 bits */ 758 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 759 uint8_t __reserved__[2]; 760 761 uint8_t __reserved__[1]; 762 uint8_t log_max_cq_sz; 763 uint8_t __reserved__[1]; 764 uint8_t log_max_cq; /* 5 bits */ 765 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 766 767 uint8_t log_max_eq_sz; 768 uint8_t log_max_mkey; /* 6 bits */ 769 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 770 uint8_t __reserved__[1]; 771 uint8_t log_max_eq; /* 4 bits */ 772 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 773 774 uint8_t max_indirection; 775 uint8_t log_max_mrw_sz; /* 7 bits */ 776 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 777 uint8_t teardown_log_max_msf_list_size; 778 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 779 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 780 0x3f 781 uint8_t log_max_klm_list_size; /* 6 bits */ 782 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 783 0x3f 784 785 uint8_t __reserved__[1]; 786 uint8_t log_max_ra_req_dc; /* 6 bits */ 787 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 788 uint8_t __reserved__[1]; 789 uint8_t log_max_ra_res_dc; /* 6 bits */ 790 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 791 0x3f 792 793 uint8_t __reserved__[1]; 794 uint8_t log_max_ra_req_qp; /* 6 bits */ 795 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 796 0x3f 797 uint8_t __reserved__[1]; 798 uint8_t log_max_ra_res_qp; /* 6 bits */ 799 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 800 0x3f 801 802 uint8_t flags1; 803 #define MCX_CAP_DEVICE_END_PAD 0x80 804 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 805 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 806 0x20 807 #define MCX_CAP_DEVICE_START_PAD 0x10 808 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 809 0x08 810 uint8_t __reserved__[1]; 811 uint16_t gid_table_size; 812 813 uint16_t flags2; 814 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 815 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 816 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 817 0x2000 818 #define MCX_CAP_DEVICE_DEBUG 0x1000 819 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 820 0x8000 821 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 822 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 823 uint16_t pkey_table_size; 824 825 uint8_t flags3; 826 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 827 0x80 828 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 829 0x40 830 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 831 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 832 #define MCX_CAP_DEVICE_ETS 0x04 833 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 834 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 835 0x01 836 uint8_t local_ca_ack_delay; /* 5 bits */ 837 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 838 0x1f 839 uint8_t port_type; 840 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 841 0x80 842 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 843 #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01 844 uint8_t num_ports; 845 846 uint8_t snapshot_log_max_msg; 847 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 848 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 849 uint8_t max_tc; /* 4 bits */ 850 #define MCX_CAP_DEVICE_MAX_TC 0x0f 851 uint8_t flags4; 852 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 853 #define MCX_CAP_DEVICE_DCBX 0x40 854 #define MCX_CAP_DEVICE_ROL_S 0x02 855 #define MCX_CAP_DEVICE_ROL_G 0x01 856 uint8_t wol; 857 #define MCX_CAP_DEVICE_WOL_S 0x40 858 #define MCX_CAP_DEVICE_WOL_G 0x20 859 #define MCX_CAP_DEVICE_WOL_A 0x10 860 #define MCX_CAP_DEVICE_WOL_B 0x08 861 #define MCX_CAP_DEVICE_WOL_M 0x04 862 #define MCX_CAP_DEVICE_WOL_U 0x02 863 #define MCX_CAP_DEVICE_WOL_P 0x01 864 865 uint16_t stat_rate_support; 866 uint8_t __reserved__[1]; 867 uint8_t cqe_version; /* 4 bits */ 868 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 869 870 uint32_t flags5; 871 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 872 0x80000000 873 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 874 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 875 0x10000000 876 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 877 0x08000000 878 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 879 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 880 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 881 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 882 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 883 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 884 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 885 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 886 #define MCX_CAP_DEVICE_SHO 0x00000100 887 #define MCX_CAP_DEVICE_TPH 0x00000080 888 #define MCX_CAP_DEVICE_RF 0x00000040 889 #define MCX_CAP_DEVICE_DCT 0x00000020 890 #define MCX_CAP_DEVICE_QOS 0x00000010 891 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 892 #define MCX_CAP_DEVICE_ROCE 0x00000004 893 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 894 895 uint32_t flags6; 896 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 897 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 898 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 899 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 900 0x10000000 901 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 902 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 903 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 904 #define MCX_CAP_DEVICE_PG 0x01000000 905 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 906 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 907 0x00400000 908 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 909 0x00200000 910 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 911 0x00100000 912 #define MCX_CAP_DEVICE_CD 0x00080000 913 #define MCX_CAP_DEVICE_ATM 0x00040000 914 #define MCX_CAP_DEVICE_APM 0x00020000 915 #define MCX_CAP_DEVICE_IMAICL 0x00010000 916 #define MCX_CAP_DEVICE_QKV 0x00000200 917 #define MCX_CAP_DEVICE_PKV 0x00000100 918 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 919 #define MCX_CAP_DEVICE_XRC 0x00000008 920 #define MCX_CAP_DEVICE_UD 0x00000004 921 #define MCX_CAP_DEVICE_UC 0x00000002 922 #define MCX_CAP_DEVICE_RC 0x00000001 923 924 uint8_t uar_flags; 925 #define MCX_CAP_DEVICE_UAR_4K 0x80 926 uint8_t uar_sz; /* 6 bits */ 927 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 928 uint8_t __reserved__[1]; 929 uint8_t log_pg_sz; 930 931 uint8_t flags7; 932 #define MCX_CAP_DEVICE_BF 0x80 933 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 934 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 935 0x20 936 uint8_t log_bf_reg_size; /* 5 bits */ 937 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 938 uint8_t __reserved__[2]; 939 940 uint16_t num_of_diagnostic_counters; 941 uint16_t max_wqe_sz_sq; 942 943 uint8_t __reserved__[2]; 944 uint16_t max_wqe_sz_rq; 945 946 uint8_t __reserved__[2]; 947 uint16_t max_wqe_sz_sq_dc; 948 949 uint32_t max_qp_mcg; /* 25 bits */ 950 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 951 952 uint8_t __reserved__[3]; 953 uint8_t log_max_mcq; 954 955 uint8_t log_max_transport_domain; /* 5 bits */ 956 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 957 0x1f 958 uint8_t log_max_pd; /* 5 bits */ 959 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 960 uint8_t __reserved__[1]; 961 uint8_t log_max_xrcd; /* 5 bits */ 962 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 963 964 uint8_t __reserved__[2]; 965 uint16_t max_flow_counter; 966 967 uint8_t log_max_rq; /* 5 bits */ 968 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 969 uint8_t log_max_sq; /* 5 bits */ 970 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 971 uint8_t log_max_tir; /* 5 bits */ 972 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 973 uint8_t log_max_tis; /* 5 bits */ 974 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 975 976 uint8_t flags8; 977 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 978 0x80 979 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 980 uint8_t log_max_rqt; /* 5 bits */ 981 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 982 uint8_t log_max_rqt_size; /* 5 bits */ 983 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 984 uint8_t log_max_tis_per_sq; /* 5 bits */ 985 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 986 0x1f 987 988 uint8_t flags9; 989 #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \ 990 0x80 991 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \ 992 0x1f 993 uint8_t log_min_stride_sz_rq; /* 5 bits */ 994 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \ 995 0x1f 996 uint8_t log_max_stride_sz_sq; /* 5 bits */ 997 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \ 998 0x1f 999 uint8_t log_min_stride_sz_sq; /* 5 bits */ 1000 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \ 1001 0x1f 1002 1003 uint8_t log_max_hairpin_queues; 1004 #define MXC_CAP_DEVICE_HAIRPIN 0x80 1005 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \ 1006 0x1f 1007 uint8_t log_min_hairpin_queues; 1008 #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \ 1009 0x1f 1010 uint8_t log_max_hairpin_num_packets; 1011 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \ 1012 0x1f 1013 uint8_t log_max_mq_sz; 1014 #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \ 1015 0x1f 1016 1017 uint8_t log_min_hairpin_wq_data_sz; 1018 #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \ 1019 0x80 1020 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \ 1021 0x40 1022 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \ 1023 0x20 1024 #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \ 1025 0x1f 1026 uint8_t log_max_vlan_list; 1027 #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \ 1028 0x80 1029 #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \ 1030 0x1f 1031 uint8_t log_max_current_mc_list; 1032 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \ 1033 0x1f 1034 uint8_t log_max_current_uc_list; 1035 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \ 1036 0x1f 1037 1038 uint8_t __reserved__[4]; 1039 1040 uint32_t create_qp_start_hint; /* 24 bits */ 1041 1042 uint8_t log_max_uctx; /* 5 bits */ 1043 #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f 1044 uint8_t log_max_umem; /* 5 bits */ 1045 #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f 1046 uint16_t max_num_eqs; 1047 1048 uint8_t log_max_l2_table; /* 5 bits */ 1049 #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f 1050 uint8_t __reserved__[1]; 1051 uint16_t log_uar_page_sz; 1052 1053 uint8_t __reserved__[8]; 1054 1055 uint32_t device_frequency_mhz; 1056 uint32_t device_frequency_khz; 1057 } __packed __aligned(8); 1058 1059 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 1060 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 1061 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 1062 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 1063 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 1064 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 1065 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98); 1066 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c); 1067 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 1068 1069 struct mcx_cmd_set_driver_version_in { 1070 uint16_t cmd_opcode; 1071 uint8_t cmd_reserved0[4]; 1072 uint16_t cmd_op_mod; 1073 uint8_t cmd_reserved1[8]; 1074 } __packed __aligned(4); 1075 1076 struct mcx_cmd_set_driver_version_out { 1077 uint8_t cmd_status; 1078 uint8_t cmd_reserved0[3]; 1079 uint32_t cmd_syndrome; 1080 uint8_t cmd_reserved1[8]; 1081 } __packed __aligned(4); 1082 1083 struct mcx_cmd_set_driver_version { 1084 uint8_t cmd_driver_version[64]; 1085 } __packed __aligned(8); 1086 1087 struct mcx_cmd_modify_nic_vport_context_in { 1088 uint16_t cmd_opcode; 1089 uint8_t cmd_reserved0[4]; 1090 uint16_t cmd_op_mod; 1091 uint8_t cmd_reserved1[4]; 1092 uint32_t cmd_field_select; 1093 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1094 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1095 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1096 } __packed __aligned(4); 1097 1098 struct mcx_cmd_modify_nic_vport_context_out { 1099 uint8_t cmd_status; 1100 uint8_t cmd_reserved0[3]; 1101 uint32_t cmd_syndrome; 1102 uint8_t cmd_reserved1[8]; 1103 } __packed __aligned(4); 1104 1105 struct mcx_cmd_query_nic_vport_context_in { 1106 uint16_t cmd_opcode; 1107 uint8_t cmd_reserved0[4]; 1108 uint16_t cmd_op_mod; 1109 uint8_t cmd_reserved1[4]; 1110 uint8_t cmd_allowed_list_type; 1111 uint8_t cmd_reserved2[3]; 1112 } __packed __aligned(4); 1113 1114 struct mcx_cmd_query_nic_vport_context_out { 1115 uint8_t cmd_status; 1116 uint8_t cmd_reserved0[3]; 1117 uint32_t cmd_syndrome; 1118 uint8_t cmd_reserved1[8]; 1119 } __packed __aligned(4); 1120 1121 struct mcx_nic_vport_ctx { 1122 uint32_t vp_min_wqe_inline_mode; 1123 uint8_t vp_reserved0[32]; 1124 uint32_t vp_mtu; 1125 uint8_t vp_reserved1[200]; 1126 uint16_t vp_flags; 1127 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1128 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1129 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1130 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1131 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1132 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1133 uint16_t vp_allowed_list_size; 1134 uint64_t vp_perm_addr; 1135 uint8_t vp_reserved2[4]; 1136 /* allowed list follows */ 1137 } __packed __aligned(4); 1138 1139 struct mcx_counter { 1140 uint64_t packets; 1141 uint64_t octets; 1142 } __packed __aligned(4); 1143 1144 struct mcx_nic_vport_counters { 1145 struct mcx_counter rx_err; 1146 struct mcx_counter tx_err; 1147 uint8_t reserved0[64]; /* 0x30 */ 1148 struct mcx_counter rx_bcast; 1149 struct mcx_counter tx_bcast; 1150 struct mcx_counter rx_ucast; 1151 struct mcx_counter tx_ucast; 1152 struct mcx_counter rx_mcast; 1153 struct mcx_counter tx_mcast; 1154 uint8_t reserved1[0x210 - 0xd0]; 1155 } __packed __aligned(4); 1156 1157 struct mcx_cmd_query_vport_counters_in { 1158 uint16_t cmd_opcode; 1159 uint8_t cmd_reserved0[4]; 1160 uint16_t cmd_op_mod; 1161 uint8_t cmd_reserved1[8]; 1162 } __packed __aligned(4); 1163 1164 struct mcx_cmd_query_vport_counters_mb_in { 1165 uint8_t cmd_reserved0[8]; 1166 uint8_t cmd_clear; 1167 uint8_t cmd_reserved1[7]; 1168 } __packed __aligned(4); 1169 1170 struct mcx_cmd_query_vport_counters_out { 1171 uint8_t cmd_status; 1172 uint8_t cmd_reserved0[3]; 1173 uint32_t cmd_syndrome; 1174 uint8_t cmd_reserved1[8]; 1175 } __packed __aligned(4); 1176 1177 struct mcx_cmd_query_flow_counter_in { 1178 uint16_t cmd_opcode; 1179 uint8_t cmd_reserved0[4]; 1180 uint16_t cmd_op_mod; 1181 uint8_t cmd_reserved1[8]; 1182 } __packed __aligned(4); 1183 1184 struct mcx_cmd_query_flow_counter_mb_in { 1185 uint8_t cmd_reserved0[8]; 1186 uint8_t cmd_clear; 1187 uint8_t cmd_reserved1[5]; 1188 uint16_t cmd_flow_counter_id; 1189 } __packed __aligned(4); 1190 1191 struct mcx_cmd_query_flow_counter_out { 1192 uint8_t cmd_status; 1193 uint8_t cmd_reserved0[3]; 1194 uint32_t cmd_syndrome; 1195 uint8_t cmd_reserved1[8]; 1196 } __packed __aligned(4); 1197 1198 struct mcx_cmd_alloc_uar_in { 1199 uint16_t cmd_opcode; 1200 uint8_t cmd_reserved0[4]; 1201 uint16_t cmd_op_mod; 1202 uint8_t cmd_reserved1[8]; 1203 } __packed __aligned(4); 1204 1205 struct mcx_cmd_alloc_uar_out { 1206 uint8_t cmd_status; 1207 uint8_t cmd_reserved0[3]; 1208 uint32_t cmd_syndrome; 1209 uint32_t cmd_uar; 1210 uint8_t cmd_reserved1[4]; 1211 } __packed __aligned(4); 1212 1213 struct mcx_cmd_query_special_ctx_in { 1214 uint16_t cmd_opcode; 1215 uint8_t cmd_reserved0[4]; 1216 uint16_t cmd_op_mod; 1217 uint8_t cmd_reserved1[8]; 1218 } __packed __aligned(4); 1219 1220 struct mcx_cmd_query_special_ctx_out { 1221 uint8_t cmd_status; 1222 uint8_t cmd_reserved0[3]; 1223 uint32_t cmd_syndrome; 1224 uint8_t cmd_reserved1[4]; 1225 uint32_t cmd_resd_lkey; 1226 } __packed __aligned(4); 1227 1228 struct mcx_eq_ctx { 1229 uint32_t eq_status; 1230 #define MCX_EQ_CTX_STATE_SHIFT 8 1231 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT) 1232 #define MCX_EQ_CTX_STATE_ARMED 0x9 1233 #define MCX_EQ_CTX_STATE_FIRED 0xa 1234 #define MCX_EQ_CTX_OI_SHIFT 17 1235 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1236 #define MCX_EQ_CTX_EC_SHIFT 18 1237 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1238 #define MCX_EQ_CTX_STATUS_SHIFT 28 1239 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1240 #define MCX_EQ_CTX_STATUS_OK 0x0 1241 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa 1242 uint32_t eq_reserved1; 1243 uint32_t eq_page_offset; 1244 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1245 uint32_t eq_uar_size; 1246 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1247 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1248 uint32_t eq_reserved2; 1249 uint8_t eq_reserved3[3]; 1250 uint8_t eq_intr; 1251 uint32_t eq_log_page_size; 1252 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1253 uint32_t eq_reserved4[3]; 1254 uint32_t eq_consumer_counter; 1255 uint32_t eq_producer_counter; 1256 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1257 uint32_t eq_reserved5[4]; 1258 } __packed __aligned(4); 1259 1260 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1261 1262 struct mcx_cmd_create_eq_in { 1263 uint16_t cmd_opcode; 1264 uint8_t cmd_reserved0[4]; 1265 uint16_t cmd_op_mod; 1266 uint8_t cmd_reserved1[8]; 1267 } __packed __aligned(4); 1268 1269 struct mcx_cmd_create_eq_mb_in { 1270 struct mcx_eq_ctx cmd_eq_ctx; 1271 uint8_t cmd_reserved0[8]; 1272 uint64_t cmd_event_bitmask; 1273 #define MCX_EVENT_TYPE_COMPLETION 0x00 1274 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1275 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1276 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1277 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1278 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1279 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1280 uint8_t cmd_reserved1[176]; 1281 } __packed __aligned(4); 1282 1283 struct mcx_cmd_create_eq_out { 1284 uint8_t cmd_status; 1285 uint8_t cmd_reserved0[3]; 1286 uint32_t cmd_syndrome; 1287 uint32_t cmd_eqn; 1288 uint8_t cmd_reserved1[4]; 1289 } __packed __aligned(4); 1290 1291 struct mcx_cmd_query_eq_in { 1292 uint16_t cmd_opcode; 1293 uint8_t cmd_reserved0[4]; 1294 uint16_t cmd_op_mod; 1295 uint32_t cmd_eqn; 1296 uint8_t cmd_reserved1[4]; 1297 } __packed __aligned(4); 1298 1299 struct mcx_cmd_query_eq_out { 1300 uint8_t cmd_status; 1301 uint8_t cmd_reserved0[3]; 1302 uint32_t cmd_syndrome; 1303 uint8_t cmd_reserved1[8]; 1304 } __packed __aligned(4); 1305 1306 struct mcx_eq_entry { 1307 uint8_t eq_reserved1; 1308 uint8_t eq_event_type; 1309 uint8_t eq_reserved2; 1310 uint8_t eq_event_sub_type; 1311 1312 uint8_t eq_reserved3[28]; 1313 uint32_t eq_event_data[7]; 1314 uint8_t eq_reserved4[2]; 1315 uint8_t eq_signature; 1316 uint8_t eq_owner; 1317 #define MCX_EQ_ENTRY_OWNER_INIT 1 1318 } __packed __aligned(4); 1319 1320 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1321 1322 struct mcx_cmd_alloc_pd_in { 1323 uint16_t cmd_opcode; 1324 uint8_t cmd_reserved0[4]; 1325 uint16_t cmd_op_mod; 1326 uint8_t cmd_reserved1[8]; 1327 } __packed __aligned(4); 1328 1329 struct mcx_cmd_alloc_pd_out { 1330 uint8_t cmd_status; 1331 uint8_t cmd_reserved0[3]; 1332 uint32_t cmd_syndrome; 1333 uint32_t cmd_pd; 1334 uint8_t cmd_reserved1[4]; 1335 } __packed __aligned(4); 1336 1337 struct mcx_cmd_alloc_td_in { 1338 uint16_t cmd_opcode; 1339 uint8_t cmd_reserved0[4]; 1340 uint16_t cmd_op_mod; 1341 uint8_t cmd_reserved1[8]; 1342 } __packed __aligned(4); 1343 1344 struct mcx_cmd_alloc_td_out { 1345 uint8_t cmd_status; 1346 uint8_t cmd_reserved0[3]; 1347 uint32_t cmd_syndrome; 1348 uint32_t cmd_tdomain; 1349 uint8_t cmd_reserved1[4]; 1350 } __packed __aligned(4); 1351 1352 struct mcx_cmd_create_tir_in { 1353 uint16_t cmd_opcode; 1354 uint8_t cmd_reserved0[4]; 1355 uint16_t cmd_op_mod; 1356 uint8_t cmd_reserved1[8]; 1357 } __packed __aligned(4); 1358 1359 struct mcx_cmd_create_tir_mb_in { 1360 uint8_t cmd_reserved0[20]; 1361 uint32_t cmd_disp_type; 1362 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1363 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1364 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1365 uint8_t cmd_reserved1[8]; 1366 uint32_t cmd_lro; 1367 uint8_t cmd_reserved2[8]; 1368 uint32_t cmd_inline_rqn; 1369 uint32_t cmd_indir_table; 1370 uint32_t cmd_tdomain; 1371 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1372 #define MCX_TIR_CTX_HASH_SHIFT 28 1373 uint8_t cmd_rx_hash_key[40]; 1374 uint32_t cmd_rx_hash_sel_outer; 1375 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1376 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1377 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1378 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1379 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1380 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1381 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1382 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1383 uint32_t cmd_rx_hash_sel_inner; 1384 uint8_t cmd_reserved3[152]; 1385 } __packed __aligned(4); 1386 1387 struct mcx_cmd_create_tir_out { 1388 uint8_t cmd_status; 1389 uint8_t cmd_reserved0[3]; 1390 uint32_t cmd_syndrome; 1391 uint32_t cmd_tirn; 1392 uint8_t cmd_reserved1[4]; 1393 } __packed __aligned(4); 1394 1395 struct mcx_cmd_destroy_tir_in { 1396 uint16_t cmd_opcode; 1397 uint8_t cmd_reserved0[4]; 1398 uint16_t cmd_op_mod; 1399 uint32_t cmd_tirn; 1400 uint8_t cmd_reserved1[4]; 1401 } __packed __aligned(4); 1402 1403 struct mcx_cmd_destroy_tir_out { 1404 uint8_t cmd_status; 1405 uint8_t cmd_reserved0[3]; 1406 uint32_t cmd_syndrome; 1407 uint8_t cmd_reserved1[8]; 1408 } __packed __aligned(4); 1409 1410 struct mcx_cmd_create_tis_in { 1411 uint16_t cmd_opcode; 1412 uint8_t cmd_reserved0[4]; 1413 uint16_t cmd_op_mod; 1414 uint8_t cmd_reserved1[8]; 1415 } __packed __aligned(4); 1416 1417 struct mcx_cmd_create_tis_mb_in { 1418 uint8_t cmd_reserved[16]; 1419 uint32_t cmd_prio; 1420 uint8_t cmd_reserved1[32]; 1421 uint32_t cmd_tdomain; 1422 uint8_t cmd_reserved2[120]; 1423 } __packed __aligned(4); 1424 1425 struct mcx_cmd_create_tis_out { 1426 uint8_t cmd_status; 1427 uint8_t cmd_reserved0[3]; 1428 uint32_t cmd_syndrome; 1429 uint32_t cmd_tisn; 1430 uint8_t cmd_reserved1[4]; 1431 } __packed __aligned(4); 1432 1433 struct mcx_cmd_destroy_tis_in { 1434 uint16_t cmd_opcode; 1435 uint8_t cmd_reserved0[4]; 1436 uint16_t cmd_op_mod; 1437 uint32_t cmd_tisn; 1438 uint8_t cmd_reserved1[4]; 1439 } __packed __aligned(4); 1440 1441 struct mcx_cmd_destroy_tis_out { 1442 uint8_t cmd_status; 1443 uint8_t cmd_reserved0[3]; 1444 uint32_t cmd_syndrome; 1445 uint8_t cmd_reserved1[8]; 1446 } __packed __aligned(4); 1447 1448 struct mcx_cmd_create_rqt_in { 1449 uint16_t cmd_opcode; 1450 uint8_t cmd_reserved0[4]; 1451 uint16_t cmd_op_mod; 1452 uint8_t cmd_reserved1[8]; 1453 } __packed __aligned(4); 1454 1455 struct mcx_rqt_ctx { 1456 uint8_t cmd_reserved0[20]; 1457 uint16_t cmd_reserved1; 1458 uint16_t cmd_rqt_max_size; 1459 uint16_t cmd_reserved2; 1460 uint16_t cmd_rqt_actual_size; 1461 uint8_t cmd_reserved3[212]; 1462 } __packed __aligned(4); 1463 1464 struct mcx_cmd_create_rqt_mb_in { 1465 uint8_t cmd_reserved0[16]; 1466 struct mcx_rqt_ctx cmd_rqt; 1467 } __packed __aligned(4); 1468 1469 struct mcx_cmd_create_rqt_out { 1470 uint8_t cmd_status; 1471 uint8_t cmd_reserved0[3]; 1472 uint32_t cmd_syndrome; 1473 uint32_t cmd_rqtn; 1474 uint8_t cmd_reserved1[4]; 1475 } __packed __aligned(4); 1476 1477 struct mcx_cmd_destroy_rqt_in { 1478 uint16_t cmd_opcode; 1479 uint8_t cmd_reserved0[4]; 1480 uint16_t cmd_op_mod; 1481 uint32_t cmd_rqtn; 1482 uint8_t cmd_reserved1[4]; 1483 } __packed __aligned(4); 1484 1485 struct mcx_cmd_destroy_rqt_out { 1486 uint8_t cmd_status; 1487 uint8_t cmd_reserved0[3]; 1488 uint32_t cmd_syndrome; 1489 uint8_t cmd_reserved1[8]; 1490 } __packed __aligned(4); 1491 1492 struct mcx_cq_ctx { 1493 uint32_t cq_status; 1494 #define MCX_CQ_CTX_STATUS_SHIFT 28 1495 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT) 1496 #define MCX_CQ_CTX_STATUS_OK 0x0 1497 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9 1498 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa 1499 #define MCX_CQ_CTX_STATE_SHIFT 8 1500 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT) 1501 #define MCX_CQ_CTX_STATE_SOLICITED 0x6 1502 #define MCX_CQ_CTX_STATE_ARMED 0x9 1503 #define MCX_CQ_CTX_STATE_FIRED 0xa 1504 uint32_t cq_reserved1; 1505 uint32_t cq_page_offset; 1506 uint32_t cq_uar_size; 1507 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1508 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1509 uint32_t cq_period_max_count; 1510 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1511 uint32_t cq_eqn; 1512 uint32_t cq_log_page_size; 1513 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1514 uint32_t cq_reserved2; 1515 uint32_t cq_last_notified; 1516 uint32_t cq_last_solicit; 1517 uint32_t cq_consumer_counter; 1518 uint32_t cq_producer_counter; 1519 uint8_t cq_reserved3[8]; 1520 uint64_t cq_doorbell; 1521 } __packed __aligned(4); 1522 1523 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1524 1525 struct mcx_cmd_create_cq_in { 1526 uint16_t cmd_opcode; 1527 uint8_t cmd_reserved0[4]; 1528 uint16_t cmd_op_mod; 1529 uint8_t cmd_reserved1[8]; 1530 } __packed __aligned(4); 1531 1532 struct mcx_cmd_create_cq_mb_in { 1533 struct mcx_cq_ctx cmd_cq_ctx; 1534 uint8_t cmd_reserved1[192]; 1535 } __packed __aligned(4); 1536 1537 struct mcx_cmd_create_cq_out { 1538 uint8_t cmd_status; 1539 uint8_t cmd_reserved0[3]; 1540 uint32_t cmd_syndrome; 1541 uint32_t cmd_cqn; 1542 uint8_t cmd_reserved1[4]; 1543 } __packed __aligned(4); 1544 1545 struct mcx_cmd_destroy_cq_in { 1546 uint16_t cmd_opcode; 1547 uint8_t cmd_reserved0[4]; 1548 uint16_t cmd_op_mod; 1549 uint32_t cmd_cqn; 1550 uint8_t cmd_reserved1[4]; 1551 } __packed __aligned(4); 1552 1553 struct mcx_cmd_destroy_cq_out { 1554 uint8_t cmd_status; 1555 uint8_t cmd_reserved0[3]; 1556 uint32_t cmd_syndrome; 1557 uint8_t cmd_reserved1[8]; 1558 } __packed __aligned(4); 1559 1560 struct mcx_cmd_query_cq_in { 1561 uint16_t cmd_opcode; 1562 uint8_t cmd_reserved0[4]; 1563 uint16_t cmd_op_mod; 1564 uint32_t cmd_cqn; 1565 uint8_t cmd_reserved1[4]; 1566 } __packed __aligned(4); 1567 1568 struct mcx_cmd_query_cq_out { 1569 uint8_t cmd_status; 1570 uint8_t cmd_reserved0[3]; 1571 uint32_t cmd_syndrome; 1572 uint8_t cmd_reserved1[8]; 1573 } __packed __aligned(4); 1574 1575 struct mcx_cq_entry { 1576 uint32_t __reserved__; 1577 uint32_t cq_lro; 1578 uint32_t cq_lro_ack_seq_num; 1579 uint32_t cq_rx_hash; 1580 uint8_t cq_rx_hash_type; 1581 uint8_t cq_ml_path; 1582 uint16_t __reserved__; 1583 uint32_t cq_checksum; 1584 uint32_t __reserved__; 1585 uint32_t cq_flags; 1586 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1587 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1588 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1589 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1590 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1591 1592 uint32_t cq_lro_srqn; 1593 uint32_t __reserved__[2]; 1594 uint32_t cq_byte_cnt; 1595 uint64_t cq_timestamp; 1596 uint8_t cq_rx_drops; 1597 uint8_t cq_flow_tag[3]; 1598 uint16_t cq_wqe_count; 1599 uint8_t cq_signature; 1600 uint8_t cq_opcode_owner; 1601 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1602 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1603 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1604 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1605 1606 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1607 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1608 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1609 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1610 1611 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1612 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1613 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1614 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1615 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1616 1617 } __packed __aligned(4); 1618 1619 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1620 1621 struct mcx_cq_doorbell { 1622 uint32_t db_update_ci; 1623 uint32_t db_arm_ci; 1624 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1625 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1626 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1627 } __packed __aligned(8); 1628 1629 struct mcx_wq_ctx { 1630 uint8_t wq_type; 1631 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1632 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1633 uint8_t wq_reserved0[5]; 1634 uint16_t wq_lwm; 1635 uint32_t wq_pd; 1636 uint32_t wq_uar_page; 1637 uint64_t wq_doorbell; 1638 uint32_t wq_hw_counter; 1639 uint32_t wq_sw_counter; 1640 uint16_t wq_log_stride; 1641 uint8_t wq_log_page_sz; 1642 uint8_t wq_log_size; 1643 uint8_t wq_reserved1[156]; 1644 } __packed __aligned(4); 1645 1646 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1647 1648 struct mcx_sq_ctx { 1649 uint32_t sq_flags; 1650 #define MCX_SQ_CTX_RLKEY (1 << 31) 1651 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1652 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1653 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1654 #define MCX_SQ_CTX_STATE_SHIFT 20 1655 #define MCX_SQ_CTX_STATE_MASK (0xf << 20) 1656 #define MCX_SQ_CTX_STATE_RST 0 1657 #define MCX_SQ_CTX_STATE_RDY 1 1658 #define MCX_SQ_CTX_STATE_ERR 3 1659 uint32_t sq_user_index; 1660 uint32_t sq_cqn; 1661 uint32_t sq_reserved1[5]; 1662 uint32_t sq_tis_lst_sz; 1663 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1664 uint32_t sq_reserved2[2]; 1665 uint32_t sq_tis_num; 1666 struct mcx_wq_ctx sq_wq; 1667 } __packed __aligned(4); 1668 1669 struct mcx_sq_entry_seg { 1670 uint32_t sqs_byte_count; 1671 uint32_t sqs_lkey; 1672 uint64_t sqs_addr; 1673 } __packed __aligned(4); 1674 1675 struct mcx_sq_entry { 1676 /* control segment */ 1677 uint32_t sqe_opcode_index; 1678 #define MCX_SQE_WQE_INDEX_SHIFT 8 1679 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1680 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1681 uint32_t sqe_ds_sq_num; 1682 #define MCX_SQE_SQ_NUM_SHIFT 8 1683 uint32_t sqe_signature; 1684 #define MCX_SQE_SIGNATURE_SHIFT 24 1685 #define MCX_SQE_SOLICITED_EVENT 0x02 1686 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1687 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1688 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1689 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1690 #define MCX_SQE_FM_NO_FENCE 0x00 1691 #define MCX_SQE_FM_SMALL_FENCE 0x40 1692 uint32_t sqe_mkey; 1693 1694 /* ethernet segment */ 1695 uint32_t sqe_reserved1; 1696 uint32_t sqe_mss_csum; 1697 #define MCX_SQE_L4_CSUM (1 << 31) 1698 #define MCX_SQE_L3_CSUM (1 << 30) 1699 uint32_t sqe_reserved2; 1700 uint16_t sqe_inline_header_size; 1701 uint16_t sqe_inline_headers[9]; 1702 1703 /* data segment */ 1704 struct mcx_sq_entry_seg sqe_segs[1]; 1705 } __packed __aligned(64); 1706 1707 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1708 1709 struct mcx_cmd_create_sq_in { 1710 uint16_t cmd_opcode; 1711 uint8_t cmd_reserved0[4]; 1712 uint16_t cmd_op_mod; 1713 uint8_t cmd_reserved1[8]; 1714 } __packed __aligned(4); 1715 1716 struct mcx_cmd_create_sq_out { 1717 uint8_t cmd_status; 1718 uint8_t cmd_reserved0[3]; 1719 uint32_t cmd_syndrome; 1720 uint32_t cmd_sqn; 1721 uint8_t cmd_reserved1[4]; 1722 } __packed __aligned(4); 1723 1724 struct mcx_cmd_modify_sq_in { 1725 uint16_t cmd_opcode; 1726 uint8_t cmd_reserved0[4]; 1727 uint16_t cmd_op_mod; 1728 uint32_t cmd_sq_state; 1729 uint8_t cmd_reserved1[4]; 1730 } __packed __aligned(4); 1731 1732 struct mcx_cmd_modify_sq_mb_in { 1733 uint32_t cmd_modify_hi; 1734 uint32_t cmd_modify_lo; 1735 uint8_t cmd_reserved0[8]; 1736 struct mcx_sq_ctx cmd_sq_ctx; 1737 } __packed __aligned(4); 1738 1739 struct mcx_cmd_modify_sq_out { 1740 uint8_t cmd_status; 1741 uint8_t cmd_reserved0[3]; 1742 uint32_t cmd_syndrome; 1743 uint8_t cmd_reserved1[8]; 1744 } __packed __aligned(4); 1745 1746 struct mcx_cmd_destroy_sq_in { 1747 uint16_t cmd_opcode; 1748 uint8_t cmd_reserved0[4]; 1749 uint16_t cmd_op_mod; 1750 uint32_t cmd_sqn; 1751 uint8_t cmd_reserved1[4]; 1752 } __packed __aligned(4); 1753 1754 struct mcx_cmd_destroy_sq_out { 1755 uint8_t cmd_status; 1756 uint8_t cmd_reserved0[3]; 1757 uint32_t cmd_syndrome; 1758 uint8_t cmd_reserved1[8]; 1759 } __packed __aligned(4); 1760 1761 1762 struct mcx_rq_ctx { 1763 uint32_t rq_flags; 1764 #define MCX_RQ_CTX_RLKEY (1 << 31) 1765 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1766 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1767 #define MCX_RQ_CTX_STATE_SHIFT 20 1768 #define MCX_RQ_CTX_STATE_MASK (0xf << 20) 1769 #define MCX_RQ_CTX_STATE_RST 0 1770 #define MCX_RQ_CTX_STATE_RDY 1 1771 #define MCX_RQ_CTX_STATE_ERR 3 1772 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1773 uint32_t rq_user_index; 1774 uint32_t rq_cqn; 1775 uint32_t rq_reserved1; 1776 uint32_t rq_rmpn; 1777 uint32_t rq_reserved2[7]; 1778 struct mcx_wq_ctx rq_wq; 1779 } __packed __aligned(4); 1780 1781 struct mcx_rq_entry { 1782 uint32_t rqe_byte_count; 1783 uint32_t rqe_lkey; 1784 uint64_t rqe_addr; 1785 } __packed __aligned(16); 1786 1787 struct mcx_cmd_create_rq_in { 1788 uint16_t cmd_opcode; 1789 uint8_t cmd_reserved0[4]; 1790 uint16_t cmd_op_mod; 1791 uint8_t cmd_reserved1[8]; 1792 } __packed __aligned(4); 1793 1794 struct mcx_cmd_create_rq_out { 1795 uint8_t cmd_status; 1796 uint8_t cmd_reserved0[3]; 1797 uint32_t cmd_syndrome; 1798 uint32_t cmd_rqn; 1799 uint8_t cmd_reserved1[4]; 1800 } __packed __aligned(4); 1801 1802 struct mcx_cmd_modify_rq_in { 1803 uint16_t cmd_opcode; 1804 uint8_t cmd_reserved0[4]; 1805 uint16_t cmd_op_mod; 1806 uint32_t cmd_rq_state; 1807 uint8_t cmd_reserved1[4]; 1808 } __packed __aligned(4); 1809 1810 struct mcx_cmd_modify_rq_mb_in { 1811 uint32_t cmd_modify_hi; 1812 uint32_t cmd_modify_lo; 1813 uint8_t cmd_reserved0[8]; 1814 struct mcx_rq_ctx cmd_rq_ctx; 1815 } __packed __aligned(4); 1816 1817 struct mcx_cmd_modify_rq_out { 1818 uint8_t cmd_status; 1819 uint8_t cmd_reserved0[3]; 1820 uint32_t cmd_syndrome; 1821 uint8_t cmd_reserved1[8]; 1822 } __packed __aligned(4); 1823 1824 struct mcx_cmd_destroy_rq_in { 1825 uint16_t cmd_opcode; 1826 uint8_t cmd_reserved0[4]; 1827 uint16_t cmd_op_mod; 1828 uint32_t cmd_rqn; 1829 uint8_t cmd_reserved1[4]; 1830 } __packed __aligned(4); 1831 1832 struct mcx_cmd_destroy_rq_out { 1833 uint8_t cmd_status; 1834 uint8_t cmd_reserved0[3]; 1835 uint32_t cmd_syndrome; 1836 uint8_t cmd_reserved1[8]; 1837 } __packed __aligned(4); 1838 1839 struct mcx_cmd_create_flow_table_in { 1840 uint16_t cmd_opcode; 1841 uint8_t cmd_reserved0[4]; 1842 uint16_t cmd_op_mod; 1843 uint8_t cmd_reserved1[8]; 1844 } __packed __aligned(4); 1845 1846 struct mcx_flow_table_ctx { 1847 uint8_t ft_miss_action; 1848 uint8_t ft_level; 1849 uint8_t ft_reserved0; 1850 uint8_t ft_log_size; 1851 uint32_t ft_table_miss_id; 1852 uint8_t ft_reserved1[28]; 1853 } __packed __aligned(4); 1854 1855 struct mcx_cmd_create_flow_table_mb_in { 1856 uint8_t cmd_table_type; 1857 uint8_t cmd_reserved0[7]; 1858 struct mcx_flow_table_ctx cmd_ctx; 1859 } __packed __aligned(4); 1860 1861 struct mcx_cmd_create_flow_table_out { 1862 uint8_t cmd_status; 1863 uint8_t cmd_reserved0[3]; 1864 uint32_t cmd_syndrome; 1865 uint32_t cmd_table_id; 1866 uint8_t cmd_reserved1[4]; 1867 } __packed __aligned(4); 1868 1869 struct mcx_cmd_destroy_flow_table_in { 1870 uint16_t cmd_opcode; 1871 uint8_t cmd_reserved0[4]; 1872 uint16_t cmd_op_mod; 1873 uint8_t cmd_reserved1[8]; 1874 } __packed __aligned(4); 1875 1876 struct mcx_cmd_destroy_flow_table_mb_in { 1877 uint8_t cmd_table_type; 1878 uint8_t cmd_reserved0[3]; 1879 uint32_t cmd_table_id; 1880 uint8_t cmd_reserved1[40]; 1881 } __packed __aligned(4); 1882 1883 struct mcx_cmd_destroy_flow_table_out { 1884 uint8_t cmd_status; 1885 uint8_t cmd_reserved0[3]; 1886 uint32_t cmd_syndrome; 1887 uint8_t cmd_reserved1[8]; 1888 } __packed __aligned(4); 1889 1890 struct mcx_cmd_set_flow_table_root_in { 1891 uint16_t cmd_opcode; 1892 uint8_t cmd_reserved0[4]; 1893 uint16_t cmd_op_mod; 1894 uint8_t cmd_reserved1[8]; 1895 } __packed __aligned(4); 1896 1897 struct mcx_cmd_set_flow_table_root_mb_in { 1898 uint8_t cmd_table_type; 1899 uint8_t cmd_reserved0[3]; 1900 uint32_t cmd_table_id; 1901 uint8_t cmd_reserved1[56]; 1902 } __packed __aligned(4); 1903 1904 struct mcx_cmd_set_flow_table_root_out { 1905 uint8_t cmd_status; 1906 uint8_t cmd_reserved0[3]; 1907 uint32_t cmd_syndrome; 1908 uint8_t cmd_reserved1[8]; 1909 } __packed __aligned(4); 1910 1911 struct mcx_flow_match { 1912 /* outer headers */ 1913 uint8_t mc_src_mac[6]; 1914 uint16_t mc_ethertype; 1915 uint8_t mc_dest_mac[6]; 1916 uint16_t mc_first_vlan; 1917 uint8_t mc_ip_proto; 1918 uint8_t mc_ip_dscp_ecn; 1919 uint8_t mc_vlan_flags; 1920 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1921 uint8_t mc_tcp_flags; 1922 uint16_t mc_tcp_sport; 1923 uint16_t mc_tcp_dport; 1924 uint32_t mc_reserved0; 1925 uint16_t mc_udp_sport; 1926 uint16_t mc_udp_dport; 1927 uint8_t mc_src_ip[16]; 1928 uint8_t mc_dest_ip[16]; 1929 1930 /* misc parameters */ 1931 uint8_t mc_reserved1[8]; 1932 uint16_t mc_second_vlan; 1933 uint8_t mc_reserved2[2]; 1934 uint8_t mc_second_vlan_flags; 1935 uint8_t mc_reserved3[15]; 1936 uint32_t mc_outer_ipv6_flow_label; 1937 uint8_t mc_reserved4[32]; 1938 1939 uint8_t mc_reserved[384]; 1940 } __packed __aligned(4); 1941 1942 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1943 1944 struct mcx_cmd_create_flow_group_in { 1945 uint16_t cmd_opcode; 1946 uint8_t cmd_reserved0[4]; 1947 uint16_t cmd_op_mod; 1948 uint8_t cmd_reserved1[8]; 1949 } __packed __aligned(4); 1950 1951 struct mcx_cmd_create_flow_group_mb_in { 1952 uint8_t cmd_table_type; 1953 uint8_t cmd_reserved0[3]; 1954 uint32_t cmd_table_id; 1955 uint8_t cmd_reserved1[4]; 1956 uint32_t cmd_start_flow_index; 1957 uint8_t cmd_reserved2[4]; 1958 uint32_t cmd_end_flow_index; 1959 uint8_t cmd_reserved3[23]; 1960 uint8_t cmd_match_criteria_enable; 1961 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1962 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1963 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1964 struct mcx_flow_match cmd_match_criteria; 1965 uint8_t cmd_reserved4[448]; 1966 } __packed __aligned(4); 1967 1968 struct mcx_cmd_create_flow_group_out { 1969 uint8_t cmd_status; 1970 uint8_t cmd_reserved0[3]; 1971 uint32_t cmd_syndrome; 1972 uint32_t cmd_group_id; 1973 uint8_t cmd_reserved1[4]; 1974 } __packed __aligned(4); 1975 1976 struct mcx_flow_ctx { 1977 uint8_t fc_reserved0[4]; 1978 uint32_t fc_group_id; 1979 uint32_t fc_flow_tag; 1980 uint32_t fc_action; 1981 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 1982 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 1983 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 1984 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 1985 uint32_t fc_dest_list_size; 1986 uint32_t fc_counter_list_size; 1987 uint8_t fc_reserved1[40]; 1988 struct mcx_flow_match fc_match_value; 1989 uint8_t fc_reserved2[192]; 1990 } __packed __aligned(4); 1991 1992 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 1993 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 1994 1995 struct mcx_cmd_destroy_flow_group_in { 1996 uint16_t cmd_opcode; 1997 uint8_t cmd_reserved0[4]; 1998 uint16_t cmd_op_mod; 1999 uint8_t cmd_reserved1[8]; 2000 } __packed __aligned(4); 2001 2002 struct mcx_cmd_destroy_flow_group_mb_in { 2003 uint8_t cmd_table_type; 2004 uint8_t cmd_reserved0[3]; 2005 uint32_t cmd_table_id; 2006 uint32_t cmd_group_id; 2007 uint8_t cmd_reserved1[36]; 2008 } __packed __aligned(4); 2009 2010 struct mcx_cmd_destroy_flow_group_out { 2011 uint8_t cmd_status; 2012 uint8_t cmd_reserved0[3]; 2013 uint32_t cmd_syndrome; 2014 uint8_t cmd_reserved1[8]; 2015 } __packed __aligned(4); 2016 2017 struct mcx_cmd_set_flow_table_entry_in { 2018 uint16_t cmd_opcode; 2019 uint8_t cmd_reserved0[4]; 2020 uint16_t cmd_op_mod; 2021 uint8_t cmd_reserved1[8]; 2022 } __packed __aligned(4); 2023 2024 struct mcx_cmd_set_flow_table_entry_mb_in { 2025 uint8_t cmd_table_type; 2026 uint8_t cmd_reserved0[3]; 2027 uint32_t cmd_table_id; 2028 uint32_t cmd_modify_enable_mask; 2029 uint8_t cmd_reserved1[4]; 2030 uint32_t cmd_flow_index; 2031 uint8_t cmd_reserved2[28]; 2032 struct mcx_flow_ctx cmd_flow_ctx; 2033 } __packed __aligned(4); 2034 2035 struct mcx_cmd_set_flow_table_entry_out { 2036 uint8_t cmd_status; 2037 uint8_t cmd_reserved0[3]; 2038 uint32_t cmd_syndrome; 2039 uint8_t cmd_reserved1[8]; 2040 } __packed __aligned(4); 2041 2042 struct mcx_cmd_query_flow_table_entry_in { 2043 uint16_t cmd_opcode; 2044 uint8_t cmd_reserved0[4]; 2045 uint16_t cmd_op_mod; 2046 uint8_t cmd_reserved1[8]; 2047 } __packed __aligned(4); 2048 2049 struct mcx_cmd_query_flow_table_entry_mb_in { 2050 uint8_t cmd_table_type; 2051 uint8_t cmd_reserved0[3]; 2052 uint32_t cmd_table_id; 2053 uint8_t cmd_reserved1[8]; 2054 uint32_t cmd_flow_index; 2055 uint8_t cmd_reserved2[28]; 2056 } __packed __aligned(4); 2057 2058 struct mcx_cmd_query_flow_table_entry_out { 2059 uint8_t cmd_status; 2060 uint8_t cmd_reserved0[3]; 2061 uint32_t cmd_syndrome; 2062 uint8_t cmd_reserved1[8]; 2063 } __packed __aligned(4); 2064 2065 struct mcx_cmd_query_flow_table_entry_mb_out { 2066 uint8_t cmd_reserved0[48]; 2067 struct mcx_flow_ctx cmd_flow_ctx; 2068 } __packed __aligned(4); 2069 2070 struct mcx_cmd_delete_flow_table_entry_in { 2071 uint16_t cmd_opcode; 2072 uint8_t cmd_reserved0[4]; 2073 uint16_t cmd_op_mod; 2074 uint8_t cmd_reserved1[8]; 2075 } __packed __aligned(4); 2076 2077 struct mcx_cmd_delete_flow_table_entry_mb_in { 2078 uint8_t cmd_table_type; 2079 uint8_t cmd_reserved0[3]; 2080 uint32_t cmd_table_id; 2081 uint8_t cmd_reserved1[8]; 2082 uint32_t cmd_flow_index; 2083 uint8_t cmd_reserved2[28]; 2084 } __packed __aligned(4); 2085 2086 struct mcx_cmd_delete_flow_table_entry_out { 2087 uint8_t cmd_status; 2088 uint8_t cmd_reserved0[3]; 2089 uint32_t cmd_syndrome; 2090 uint8_t cmd_reserved1[8]; 2091 } __packed __aligned(4); 2092 2093 struct mcx_cmd_query_flow_group_in { 2094 uint16_t cmd_opcode; 2095 uint8_t cmd_reserved0[4]; 2096 uint16_t cmd_op_mod; 2097 uint8_t cmd_reserved1[8]; 2098 } __packed __aligned(4); 2099 2100 struct mcx_cmd_query_flow_group_mb_in { 2101 uint8_t cmd_table_type; 2102 uint8_t cmd_reserved0[3]; 2103 uint32_t cmd_table_id; 2104 uint32_t cmd_group_id; 2105 uint8_t cmd_reserved1[36]; 2106 } __packed __aligned(4); 2107 2108 struct mcx_cmd_query_flow_group_out { 2109 uint8_t cmd_status; 2110 uint8_t cmd_reserved0[3]; 2111 uint32_t cmd_syndrome; 2112 uint8_t cmd_reserved1[8]; 2113 } __packed __aligned(4); 2114 2115 struct mcx_cmd_query_flow_group_mb_out { 2116 uint8_t cmd_reserved0[12]; 2117 uint32_t cmd_start_flow_index; 2118 uint8_t cmd_reserved1[4]; 2119 uint32_t cmd_end_flow_index; 2120 uint8_t cmd_reserved2[20]; 2121 uint32_t cmd_match_criteria_enable; 2122 uint8_t cmd_match_criteria[512]; 2123 uint8_t cmd_reserved4[448]; 2124 } __packed __aligned(4); 2125 2126 struct mcx_cmd_query_flow_table_in { 2127 uint16_t cmd_opcode; 2128 uint8_t cmd_reserved0[4]; 2129 uint16_t cmd_op_mod; 2130 uint8_t cmd_reserved1[8]; 2131 } __packed __aligned(4); 2132 2133 struct mcx_cmd_query_flow_table_mb_in { 2134 uint8_t cmd_table_type; 2135 uint8_t cmd_reserved0[3]; 2136 uint32_t cmd_table_id; 2137 uint8_t cmd_reserved1[40]; 2138 } __packed __aligned(4); 2139 2140 struct mcx_cmd_query_flow_table_out { 2141 uint8_t cmd_status; 2142 uint8_t cmd_reserved0[3]; 2143 uint32_t cmd_syndrome; 2144 uint8_t cmd_reserved1[8]; 2145 } __packed __aligned(4); 2146 2147 struct mcx_cmd_query_flow_table_mb_out { 2148 uint8_t cmd_reserved0[4]; 2149 struct mcx_flow_table_ctx cmd_ctx; 2150 } __packed __aligned(4); 2151 2152 struct mcx_cmd_alloc_flow_counter_in { 2153 uint16_t cmd_opcode; 2154 uint8_t cmd_reserved0[4]; 2155 uint16_t cmd_op_mod; 2156 uint8_t cmd_reserved1[8]; 2157 } __packed __aligned(4); 2158 2159 struct mcx_cmd_query_rq_in { 2160 uint16_t cmd_opcode; 2161 uint8_t cmd_reserved0[4]; 2162 uint16_t cmd_op_mod; 2163 uint32_t cmd_rqn; 2164 uint8_t cmd_reserved1[4]; 2165 } __packed __aligned(4); 2166 2167 struct mcx_cmd_query_rq_out { 2168 uint8_t cmd_status; 2169 uint8_t cmd_reserved0[3]; 2170 uint32_t cmd_syndrome; 2171 uint8_t cmd_reserved1[8]; 2172 } __packed __aligned(4); 2173 2174 struct mcx_cmd_query_rq_mb_out { 2175 uint8_t cmd_reserved0[16]; 2176 struct mcx_rq_ctx cmd_ctx; 2177 }; 2178 2179 struct mcx_cmd_query_sq_in { 2180 uint16_t cmd_opcode; 2181 uint8_t cmd_reserved0[4]; 2182 uint16_t cmd_op_mod; 2183 uint32_t cmd_sqn; 2184 uint8_t cmd_reserved1[4]; 2185 } __packed __aligned(4); 2186 2187 struct mcx_cmd_query_sq_out { 2188 uint8_t cmd_status; 2189 uint8_t cmd_reserved0[3]; 2190 uint32_t cmd_syndrome; 2191 uint8_t cmd_reserved1[8]; 2192 } __packed __aligned(4); 2193 2194 struct mcx_cmd_query_sq_mb_out { 2195 uint8_t cmd_reserved0[16]; 2196 struct mcx_sq_ctx cmd_ctx; 2197 }; 2198 2199 struct mcx_cmd_alloc_flow_counter_out { 2200 uint8_t cmd_status; 2201 uint8_t cmd_reserved0[3]; 2202 uint32_t cmd_syndrome; 2203 uint8_t cmd_reserved1[2]; 2204 uint16_t cmd_flow_counter_id; 2205 uint8_t cmd_reserved2[4]; 2206 } __packed __aligned(4); 2207 2208 struct mcx_wq_doorbell { 2209 uint32_t db_recv_counter; 2210 uint32_t db_send_counter; 2211 } __packed __aligned(8); 2212 2213 struct mcx_dmamem { 2214 bus_dmamap_t mxm_map; 2215 bus_dma_segment_t mxm_seg; 2216 int mxm_nsegs; 2217 size_t mxm_size; 2218 caddr_t mxm_kva; 2219 }; 2220 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2221 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2222 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2223 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2224 2225 struct mcx_hwmem { 2226 bus_dmamap_t mhm_map; 2227 bus_dma_segment_t *mhm_segs; 2228 unsigned int mhm_seg_count; 2229 unsigned int mhm_npages; 2230 }; 2231 2232 struct mcx_slot { 2233 bus_dmamap_t ms_map; 2234 struct mbuf *ms_m; 2235 }; 2236 2237 struct mcx_eq { 2238 int eq_n; 2239 uint32_t eq_cons; 2240 struct mcx_dmamem eq_mem; 2241 }; 2242 2243 struct mcx_cq { 2244 int cq_n; 2245 struct mcx_dmamem cq_mem; 2246 uint32_t *cq_doorbell; 2247 uint32_t cq_cons; 2248 uint32_t cq_count; 2249 }; 2250 2251 struct mcx_calibration { 2252 uint64_t c_timestamp; /* previous mcx chip time */ 2253 uint64_t c_uptime; /* previous kernel nanouptime */ 2254 uint64_t c_tbase; /* mcx chip time */ 2255 uint64_t c_ubase; /* kernel nanouptime */ 2256 uint64_t c_ratio; 2257 }; 2258 2259 #define MCX_CALIBRATE_FIRST 2 2260 #define MCX_CALIBRATE_NORMAL 32 2261 2262 struct mcx_rx { 2263 struct mcx_softc *rx_softc; 2264 struct ifiqueue *rx_ifiq; 2265 2266 int rx_rqn; 2267 struct mcx_dmamem rx_rq_mem; 2268 struct mcx_slot *rx_slots; 2269 uint32_t *rx_doorbell; 2270 2271 uint32_t rx_prod; 2272 struct timeout rx_refill; 2273 struct if_rxring rx_rxr; 2274 } __aligned(64); 2275 2276 struct mcx_tx { 2277 struct mcx_softc *tx_softc; 2278 struct ifqueue *tx_ifq; 2279 2280 int tx_uar; 2281 int tx_sqn; 2282 struct mcx_dmamem tx_sq_mem; 2283 struct mcx_slot *tx_slots; 2284 uint32_t *tx_doorbell; 2285 int tx_bf_offset; 2286 2287 uint32_t tx_cons; 2288 uint32_t tx_prod; 2289 } __aligned(64); 2290 2291 struct mcx_queues { 2292 char q_name[16]; 2293 void *q_ihc; 2294 struct mcx_softc *q_sc; 2295 int q_uar; 2296 int q_index; 2297 struct mcx_rx q_rx; 2298 struct mcx_tx q_tx; 2299 struct mcx_cq q_cq; 2300 struct mcx_eq q_eq; 2301 #if NKSTAT > 0 2302 struct kstat *q_kstat; 2303 #endif 2304 }; 2305 2306 struct mcx_flow_group { 2307 int g_id; 2308 int g_table; 2309 int g_start; 2310 int g_size; 2311 }; 2312 2313 #define MCX_FLOW_GROUP_PROMISC 0 2314 #define MCX_FLOW_GROUP_ALLMULTI 1 2315 #define MCX_FLOW_GROUP_MAC 2 2316 #define MCX_FLOW_GROUP_RSS_L4 3 2317 #define MCX_FLOW_GROUP_RSS_L3 4 2318 #define MCX_FLOW_GROUP_RSS_NONE 5 2319 #define MCX_NUM_FLOW_GROUPS 6 2320 2321 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2322 MCX_TIR_CTX_HASH_SEL_DST_IP 2323 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2324 MCX_TIR_CTX_HASH_SEL_DPORT 2325 2326 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2327 MCX_TIR_CTX_HASH_SEL_IPV4 2328 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2329 MCX_TIR_CTX_HASH_SEL_IPV6 2330 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2331 MCX_TIR_CTX_HASH_SEL_IPV4 2332 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2333 MCX_TIR_CTX_HASH_SEL_IPV6 2334 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2335 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2336 2337 /* 2338 * There are a few different pieces involved in configuring RSS. 2339 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2340 * different rx queues based on a hash value. We only create one, because 2341 * we want to scatter any traffic we can apply RSS to across all our rx 2342 * queues. Anything else will only be delivered to the first rx queue, 2343 * which doesn't require an RQT. 2344 * 2345 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2346 * queue or an RQT, and in the latter case, specifies the set of fields 2347 * hashed, the hash function, and the hash key. We need one of these for each 2348 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2349 * and one for non-RSS traffic. 2350 * 2351 * Flow tables hold flow table entries in sequence. The first entry that 2352 * matches a packet is applied, sending the packet to either another flow 2353 * table or a TIR. We use one flow table to select packets based on 2354 * destination MAC address, and a second to apply RSS. The entries in the 2355 * first table send matching packets to the second, and the entries in the 2356 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2357 * 2358 * The flow table entry that delivers packets to an RSS TIR must include match 2359 * criteria that ensure packets delivered to the TIR include all the fields 2360 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2361 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2362 * each TIR. 2363 * 2364 * All of this is a lot more flexible than we need, and we can describe most 2365 * of the stuff we need with a simple array. 2366 * 2367 * An RSS config creates a TIR with hashing enabled on a set of fields, 2368 * pointing to either the first rx queue or the RQT containing all the rx 2369 * queues, and a flow table entry that matches on an ether type and 2370 * optionally an ip proto, that delivers packets to the TIR. 2371 */ 2372 static struct mcx_rss_rule { 2373 int hash_sel; 2374 int flow_group; 2375 int ethertype; 2376 int ip_proto; 2377 } mcx_rss_config[] = { 2378 /* udp and tcp for v4/v6 */ 2379 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2380 ETHERTYPE_IP, IPPROTO_TCP }, 2381 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2382 ETHERTYPE_IPV6, IPPROTO_TCP }, 2383 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2384 ETHERTYPE_IP, IPPROTO_UDP }, 2385 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2386 ETHERTYPE_IPV6, IPPROTO_UDP }, 2387 2388 /* other v4/v6 */ 2389 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2390 ETHERTYPE_IP, 0 }, 2391 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2392 ETHERTYPE_IPV6, 0 }, 2393 2394 /* non v4/v6 */ 2395 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2396 }; 2397 2398 struct mcx_softc { 2399 struct device sc_dev; 2400 struct arpcom sc_ac; 2401 struct ifmedia sc_media; 2402 uint64_t sc_media_status; 2403 uint64_t sc_media_active; 2404 2405 pci_chipset_tag_t sc_pc; 2406 pci_intr_handle_t sc_ih; 2407 void *sc_ihc; 2408 pcitag_t sc_tag; 2409 2410 bus_dma_tag_t sc_dmat; 2411 bus_space_tag_t sc_memt; 2412 bus_space_handle_t sc_memh; 2413 bus_size_t sc_mems; 2414 2415 struct mcx_dmamem sc_cmdq_mem; 2416 unsigned int sc_cmdq_mask; 2417 unsigned int sc_cmdq_size; 2418 2419 unsigned int sc_cmdq_token; 2420 2421 struct mcx_hwmem sc_boot_pages; 2422 struct mcx_hwmem sc_init_pages; 2423 struct mcx_hwmem sc_regular_pages; 2424 2425 int sc_uar; 2426 int sc_pd; 2427 int sc_tdomain; 2428 uint32_t sc_lkey; 2429 int sc_tis; 2430 int sc_tir[nitems(mcx_rss_config)]; 2431 int sc_rqt; 2432 2433 struct mcx_dmamem sc_doorbell_mem; 2434 2435 struct mcx_eq sc_admin_eq; 2436 struct mcx_eq sc_queue_eq; 2437 2438 int sc_hardmtu; 2439 int sc_rxbufsz; 2440 2441 int sc_bf_size; 2442 int sc_max_rqt_size; 2443 2444 struct task sc_port_change; 2445 2446 int sc_mac_flow_table_id; 2447 int sc_rss_flow_table_id; 2448 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2449 int sc_promisc_flow_enabled; 2450 int sc_allmulti_flow_enabled; 2451 int sc_mcast_flow_base; 2452 int sc_extra_mcast; 2453 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2454 2455 struct mcx_calibration sc_calibration[2]; 2456 unsigned int sc_calibration_gen; 2457 struct timeout sc_calibrate; 2458 uint32_t sc_mhz; 2459 uint32_t sc_khz; 2460 2461 struct mcx_queues sc_queues[MCX_MAX_QUEUES]; 2462 unsigned int sc_nqueues; 2463 2464 #if NKSTAT > 0 2465 struct kstat *sc_kstat_ieee8023; 2466 struct kstat *sc_kstat_rfc2863; 2467 struct kstat *sc_kstat_rfc2819; 2468 struct kstat *sc_kstat_rfc3635; 2469 unsigned int sc_kstat_mtmp_count; 2470 struct kstat **sc_kstat_mtmp; 2471 #endif 2472 2473 struct timecounter sc_timecounter; 2474 }; 2475 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2476 2477 static int mcx_match(struct device *, void *, void *); 2478 static void mcx_attach(struct device *, struct device *, void *); 2479 2480 #if NKSTAT > 0 2481 static void mcx_kstat_attach(struct mcx_softc *); 2482 #endif 2483 2484 static void mcx_timecounter_attach(struct mcx_softc *); 2485 2486 static int mcx_version(struct mcx_softc *); 2487 static int mcx_init_wait(struct mcx_softc *); 2488 static int mcx_enable_hca(struct mcx_softc *); 2489 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2490 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2491 int); 2492 static int mcx_issi(struct mcx_softc *); 2493 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2494 static int mcx_hca_max_caps(struct mcx_softc *); 2495 static int mcx_hca_set_caps(struct mcx_softc *); 2496 static int mcx_init_hca(struct mcx_softc *); 2497 static int mcx_set_driver_version(struct mcx_softc *); 2498 static int mcx_iff(struct mcx_softc *); 2499 static int mcx_alloc_uar(struct mcx_softc *, int *); 2500 static int mcx_alloc_pd(struct mcx_softc *); 2501 static int mcx_alloc_tdomain(struct mcx_softc *); 2502 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2503 uint64_t, int); 2504 static int mcx_query_nic_vport_context(struct mcx_softc *); 2505 static int mcx_query_special_contexts(struct mcx_softc *); 2506 static int mcx_set_port_mtu(struct mcx_softc *, int); 2507 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2508 int); 2509 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2510 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2511 int); 2512 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2513 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2514 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2515 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2516 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2517 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2518 int *); 2519 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2520 int *); 2521 static int mcx_destroy_tir(struct mcx_softc *, int); 2522 static int mcx_create_tis(struct mcx_softc *, int *); 2523 static int mcx_destroy_tis(struct mcx_softc *, int); 2524 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2525 static int mcx_destroy_rqt(struct mcx_softc *, int); 2526 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2527 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2528 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2529 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2530 int, int, struct mcx_flow_match *); 2531 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2532 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2533 uint8_t *, uint32_t); 2534 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2535 int, int, uint32_t); 2536 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2537 2538 #if NKSTAT > 0 2539 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *); 2540 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *); 2541 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *); 2542 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *); 2543 #endif 2544 2545 #if 0 2546 static int mcx_dump_flow_table(struct mcx_softc *, int); 2547 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2548 static int mcx_dump_flow_group(struct mcx_softc *, int); 2549 #endif 2550 2551 2552 /* 2553 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2554 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2555 */ 2556 static void mcx_refill(void *); 2557 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2558 struct mcx_cq_entry *, struct mbuf_list *, 2559 const struct mcx_calibration *); 2560 static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2561 struct mcx_cq_entry *); 2562 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2563 struct mcx_cq *); 2564 2565 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2566 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2567 static int mcx_admin_intr(void *); 2568 static int mcx_cq_intr(void *); 2569 2570 static int mcx_up(struct mcx_softc *); 2571 static void mcx_down(struct mcx_softc *); 2572 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2573 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2574 static void mcx_start(struct ifqueue *); 2575 static void mcx_watchdog(struct ifnet *); 2576 static void mcx_media_add_types(struct mcx_softc *); 2577 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2578 static int mcx_media_change(struct ifnet *); 2579 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2580 static void mcx_port_change(void *); 2581 2582 static void mcx_calibrate_first(struct mcx_softc *); 2583 static void mcx_calibrate(void *); 2584 2585 static inline uint32_t 2586 mcx_rd(struct mcx_softc *, bus_size_t); 2587 static inline void 2588 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2589 static inline void 2590 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2591 2592 static uint64_t mcx_timer(struct mcx_softc *); 2593 2594 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2595 bus_size_t, u_int align); 2596 static void mcx_dmamem_zero(struct mcx_dmamem *); 2597 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2598 2599 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2600 unsigned int); 2601 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2602 2603 struct cfdriver mcx_cd = { 2604 NULL, 2605 "mcx", 2606 DV_IFNET, 2607 }; 2608 2609 struct cfattach mcx_ca = { 2610 sizeof(struct mcx_softc), 2611 mcx_match, 2612 mcx_attach, 2613 }; 2614 2615 static const struct pci_matchid mcx_devices[] = { 2616 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2617 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2618 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2619 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2620 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2621 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2622 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2623 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2624 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 }, 2625 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2626 }; 2627 2628 struct mcx_eth_proto_capability { 2629 uint64_t cap_media; 2630 uint64_t cap_baudrate; 2631 }; 2632 2633 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2634 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2635 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2636 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2637 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2638 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2639 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2640 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2641 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2642 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2643 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2644 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2645 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2646 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2647 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2648 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2649 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2650 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2651 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2652 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2653 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2654 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2655 }; 2656 2657 static int 2658 mcx_get_id(uint32_t val) 2659 { 2660 return betoh32(val) & 0x00ffffff; 2661 } 2662 2663 static int 2664 mcx_match(struct device *parent, void *match, void *aux) 2665 { 2666 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2667 } 2668 2669 void 2670 mcx_attach(struct device *parent, struct device *self, void *aux) 2671 { 2672 struct mcx_softc *sc = (struct mcx_softc *)self; 2673 struct ifnet *ifp = &sc->sc_ac.ac_if; 2674 struct pci_attach_args *pa = aux; 2675 pcireg_t memtype; 2676 uint32_t r; 2677 unsigned int cq_stride; 2678 unsigned int cq_size; 2679 const char *intrstr; 2680 int i, msix; 2681 2682 sc->sc_pc = pa->pa_pc; 2683 sc->sc_tag = pa->pa_tag; 2684 sc->sc_dmat = pa->pa_dmat; 2685 2686 /* Map the PCI memory space */ 2687 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2688 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2689 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2690 NULL, &sc->sc_mems, 0)) { 2691 printf(": unable to map register memory\n"); 2692 return; 2693 } 2694 2695 if (mcx_version(sc) != 0) { 2696 /* error printed by mcx_version */ 2697 goto unmap; 2698 } 2699 2700 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2701 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2702 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2703 if (cq_size > MCX_MAX_CQE) { 2704 printf(", command queue size overflow %u\n", cq_size); 2705 goto unmap; 2706 } 2707 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2708 printf(", command queue entry size underflow %u\n", cq_stride); 2709 goto unmap; 2710 } 2711 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2712 printf(", command queue page overflow\n"); 2713 goto unmap; 2714 } 2715 2716 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2717 MCX_PAGE_SIZE) != 0) { 2718 printf(", unable to allocate doorbell memory\n"); 2719 goto unmap; 2720 } 2721 2722 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2723 MCX_PAGE_SIZE) != 0) { 2724 printf(", unable to allocate command queue\n"); 2725 goto dbfree; 2726 } 2727 2728 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2729 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2730 BUS_SPACE_BARRIER_WRITE); 2731 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2732 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2733 BUS_SPACE_BARRIER_WRITE); 2734 2735 if (mcx_init_wait(sc) != 0) { 2736 printf(", timeout waiting for init\n"); 2737 goto cqfree; 2738 } 2739 2740 sc->sc_cmdq_mask = cq_size - 1; 2741 sc->sc_cmdq_size = cq_stride; 2742 2743 if (mcx_enable_hca(sc) != 0) { 2744 /* error printed by mcx_enable_hca */ 2745 goto cqfree; 2746 } 2747 2748 if (mcx_issi(sc) != 0) { 2749 /* error printed by mcx_issi */ 2750 goto teardown; 2751 } 2752 2753 if (mcx_pages(sc, &sc->sc_boot_pages, 2754 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2755 /* error printed by mcx_pages */ 2756 goto teardown; 2757 } 2758 2759 if (mcx_hca_max_caps(sc) != 0) { 2760 /* error printed by mcx_hca_max_caps */ 2761 goto teardown; 2762 } 2763 2764 if (mcx_hca_set_caps(sc) != 0) { 2765 /* error printed by mcx_hca_set_caps */ 2766 goto teardown; 2767 } 2768 2769 if (mcx_pages(sc, &sc->sc_init_pages, 2770 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2771 /* error printed by mcx_pages */ 2772 goto teardown; 2773 } 2774 2775 if (mcx_init_hca(sc) != 0) { 2776 /* error printed by mcx_init_hca */ 2777 goto teardown; 2778 } 2779 2780 if (mcx_pages(sc, &sc->sc_regular_pages, 2781 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2782 /* error printed by mcx_pages */ 2783 goto teardown; 2784 } 2785 2786 /* apparently not necessary? */ 2787 if (mcx_set_driver_version(sc) != 0) { 2788 /* error printed by mcx_set_driver_version */ 2789 goto teardown; 2790 } 2791 2792 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2793 /* error printed by mcx_iff? */ 2794 goto teardown; 2795 } 2796 2797 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2798 /* error printed by mcx_alloc_uar */ 2799 goto teardown; 2800 } 2801 2802 if (mcx_alloc_pd(sc) != 0) { 2803 /* error printed by mcx_alloc_pd */ 2804 goto teardown; 2805 } 2806 2807 if (mcx_alloc_tdomain(sc) != 0) { 2808 /* error printed by mcx_alloc_tdomain */ 2809 goto teardown; 2810 } 2811 2812 /* 2813 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2814 * mellanox support tells me legacy interrupts are not supported, 2815 * so we're stuck with just msi-x. 2816 */ 2817 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2818 printf(": unable to map interrupt\n"); 2819 goto teardown; 2820 } 2821 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2822 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2823 IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc)); 2824 if (sc->sc_ihc == NULL) { 2825 printf(": unable to establish interrupt"); 2826 if (intrstr != NULL) 2827 printf(" at %s", intrstr); 2828 printf("\n"); 2829 goto teardown; 2830 } 2831 2832 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2833 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2834 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2835 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2836 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2837 /* error printed by mcx_create_eq */ 2838 goto teardown; 2839 } 2840 2841 if (mcx_query_nic_vport_context(sc) != 0) { 2842 /* error printed by mcx_query_nic_vport_context */ 2843 goto teardown; 2844 } 2845 2846 if (mcx_query_special_contexts(sc) != 0) { 2847 /* error printed by mcx_query_special_contexts */ 2848 goto teardown; 2849 } 2850 2851 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2852 /* error printed by mcx_set_port_mtu */ 2853 goto teardown; 2854 } 2855 2856 printf(", %s, address %s\n", intrstr, 2857 ether_sprintf(sc->sc_ac.ac_enaddr)); 2858 2859 msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); 2860 sc->sc_nqueues = 1; 2861 2862 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2863 ifp->if_softc = sc; 2864 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2865 ifp->if_xflags = IFXF_MPSAFE; 2866 ifp->if_ioctl = mcx_ioctl; 2867 ifp->if_qstart = mcx_start; 2868 ifp->if_watchdog = mcx_watchdog; 2869 ifp->if_hardmtu = sc->sc_hardmtu; 2870 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 | 2871 IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 | 2872 IFCAP_CSUM_TCPv6; 2873 #if NVLAN > 0 2874 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2875 #endif 2876 ifq_set_maxlen(&ifp->if_snd, 1024); 2877 2878 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2879 mcx_media_status); 2880 mcx_media_add_types(sc); 2881 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2882 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2883 2884 if_attach(ifp); 2885 ether_ifattach(ifp); 2886 2887 if_attach_iqueues(ifp, sc->sc_nqueues); 2888 if_attach_queues(ifp, sc->sc_nqueues); 2889 for (i = 0; i < sc->sc_nqueues; i++) { 2890 struct ifiqueue *ifiq = ifp->if_iqs[i]; 2891 struct ifqueue *ifq = ifp->if_ifqs[i]; 2892 struct mcx_queues *q = &sc->sc_queues[i]; 2893 struct mcx_rx *rx = &q->q_rx; 2894 struct mcx_tx *tx = &q->q_tx; 2895 pci_intr_handle_t ih; 2896 int vec; 2897 2898 vec = i + 1; 2899 q->q_sc = sc; 2900 q->q_index = i; 2901 2902 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 2903 printf("%s: unable to alloc uar %d\n", 2904 DEVNAME(sc), i); 2905 goto teardown; 2906 } 2907 2908 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 2909 printf("%s: unable to create event queue %d\n", 2910 DEVNAME(sc), i); 2911 goto teardown; 2912 } 2913 2914 rx->rx_softc = sc; 2915 rx->rx_ifiq = ifiq; 2916 timeout_set(&rx->rx_refill, mcx_refill, rx); 2917 ifiq->ifiq_softc = rx; 2918 2919 tx->tx_softc = sc; 2920 tx->tx_ifq = ifq; 2921 ifq->ifq_softc = tx; 2922 2923 if (pci_intr_map_msix(pa, vec, &ih) != 0) { 2924 printf("%s: unable to map queue interrupt %d\n", 2925 DEVNAME(sc), i); 2926 goto teardown; 2927 } 2928 snprintf(q->q_name, sizeof(q->q_name), "%s:%d", 2929 DEVNAME(sc), i); 2930 q->q_ihc = pci_intr_establish(sc->sc_pc, ih, 2931 IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); 2932 } 2933 2934 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2935 2936 task_set(&sc->sc_port_change, mcx_port_change, sc); 2937 mcx_port_change(sc); 2938 2939 sc->sc_mac_flow_table_id = -1; 2940 sc->sc_rss_flow_table_id = -1; 2941 sc->sc_rqt = -1; 2942 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 2943 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 2944 mfg->g_id = -1; 2945 mfg->g_table = -1; 2946 mfg->g_size = 0; 2947 mfg->g_start = 0; 2948 } 2949 sc->sc_extra_mcast = 0; 2950 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 2951 2952 #if NKSTAT > 0 2953 mcx_kstat_attach(sc); 2954 #endif 2955 mcx_timecounter_attach(sc); 2956 return; 2957 2958 teardown: 2959 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 2960 /* error printed by mcx_teardown_hca, and we're already unwinding */ 2961 cqfree: 2962 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2963 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2964 BUS_SPACE_BARRIER_WRITE); 2965 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 2966 MCX_CMDQ_INTERFACE_DISABLED); 2967 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 2968 BUS_SPACE_BARRIER_WRITE); 2969 2970 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 2971 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2972 BUS_SPACE_BARRIER_WRITE); 2973 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 2974 2975 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 2976 dbfree: 2977 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 2978 unmap: 2979 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 2980 sc->sc_mems = 0; 2981 } 2982 2983 static int 2984 mcx_version(struct mcx_softc *sc) 2985 { 2986 uint32_t fw0, fw1; 2987 uint16_t cmdif; 2988 2989 fw0 = mcx_rd(sc, MCX_FW_VER); 2990 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 2991 2992 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 2993 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 2994 2995 cmdif = MCX_CMDIF(fw1); 2996 if (cmdif != MCX_CMD_IF_SUPPORTED) { 2997 printf(", unsupported command interface %u\n", cmdif); 2998 return (-1); 2999 } 3000 3001 return (0); 3002 } 3003 3004 static int 3005 mcx_init_wait(struct mcx_softc *sc) 3006 { 3007 unsigned int i; 3008 uint32_t r; 3009 3010 for (i = 0; i < 2000; i++) { 3011 r = mcx_rd(sc, MCX_STATE); 3012 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 3013 return (0); 3014 3015 delay(1000); 3016 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 3017 BUS_SPACE_BARRIER_READ); 3018 } 3019 3020 return (-1); 3021 } 3022 3023 static uint8_t 3024 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3025 unsigned int msec) 3026 { 3027 unsigned int i; 3028 3029 for (i = 0; i < msec; i++) { 3030 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3031 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3032 3033 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 3034 MCX_CQ_STATUS_OWN_SW) 3035 return (0); 3036 3037 delay(1000); 3038 } 3039 3040 return (ETIMEDOUT); 3041 } 3042 3043 static uint32_t 3044 mcx_mix_u64(uint32_t xor, uint64_t u64) 3045 { 3046 xor ^= u64 >> 32; 3047 xor ^= u64; 3048 3049 return (xor); 3050 } 3051 3052 static uint32_t 3053 mcx_mix_u32(uint32_t xor, uint32_t u32) 3054 { 3055 xor ^= u32; 3056 3057 return (xor); 3058 } 3059 3060 static uint32_t 3061 mcx_mix_u8(uint32_t xor, uint8_t u8) 3062 { 3063 xor ^= u8; 3064 3065 return (xor); 3066 } 3067 3068 static uint8_t 3069 mcx_mix_done(uint32_t xor) 3070 { 3071 xor ^= xor >> 16; 3072 xor ^= xor >> 8; 3073 3074 return (xor); 3075 } 3076 3077 static uint8_t 3078 mcx_xor(const void *buf, size_t len) 3079 { 3080 const uint32_t *dwords = buf; 3081 uint32_t xor = 0xff; 3082 size_t i; 3083 3084 len /= sizeof(*dwords); 3085 3086 for (i = 0; i < len; i++) 3087 xor ^= dwords[i]; 3088 3089 return (mcx_mix_done(xor)); 3090 } 3091 3092 static uint8_t 3093 mcx_cmdq_token(struct mcx_softc *sc) 3094 { 3095 uint8_t token; 3096 3097 do { 3098 token = ++sc->sc_cmdq_token; 3099 } while (token == 0); 3100 3101 return (token); 3102 } 3103 3104 static void 3105 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3106 uint32_t ilen, uint32_t olen, uint8_t token) 3107 { 3108 memset(cqe, 0, sc->sc_cmdq_size); 3109 3110 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 3111 htobem32(&cqe->cq_input_length, ilen); 3112 htobem32(&cqe->cq_output_length, olen); 3113 cqe->cq_token = token; 3114 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 3115 } 3116 3117 static void 3118 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 3119 { 3120 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 3121 } 3122 3123 static int 3124 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 3125 { 3126 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 3127 return (0); 3128 } 3129 3130 static void * 3131 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 3132 { 3133 return (&cqe->cq_input_data); 3134 } 3135 3136 static void * 3137 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3138 { 3139 return (&cqe->cq_output_data); 3140 } 3141 3142 static void 3143 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3144 unsigned int slot) 3145 { 3146 mcx_cmdq_sign(cqe); 3147 3148 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3149 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3150 3151 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3152 mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t), 3153 BUS_SPACE_BARRIER_WRITE); 3154 } 3155 3156 static int 3157 mcx_enable_hca(struct mcx_softc *sc) 3158 { 3159 struct mcx_cmdq_entry *cqe; 3160 struct mcx_cmd_enable_hca_in *in; 3161 struct mcx_cmd_enable_hca_out *out; 3162 int error; 3163 uint8_t status; 3164 3165 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3166 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3167 3168 in = mcx_cmdq_in(cqe); 3169 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3170 in->cmd_op_mod = htobe16(0); 3171 in->cmd_function_id = htobe16(0); 3172 3173 mcx_cmdq_post(sc, cqe, 0); 3174 3175 error = mcx_cmdq_poll(sc, cqe, 1000); 3176 if (error != 0) { 3177 printf(", hca enable timeout\n"); 3178 return (-1); 3179 } 3180 if (mcx_cmdq_verify(cqe) != 0) { 3181 printf(", hca enable command corrupt\n"); 3182 return (-1); 3183 } 3184 3185 status = cqe->cq_output_data[0]; 3186 if (status != MCX_CQ_STATUS_OK) { 3187 printf(", hca enable failed (%x)\n", status); 3188 return (-1); 3189 } 3190 3191 return (0); 3192 } 3193 3194 static int 3195 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3196 { 3197 struct mcx_cmdq_entry *cqe; 3198 struct mcx_cmd_teardown_hca_in *in; 3199 struct mcx_cmd_teardown_hca_out *out; 3200 int error; 3201 uint8_t status; 3202 3203 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3204 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3205 3206 in = mcx_cmdq_in(cqe); 3207 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3208 in->cmd_op_mod = htobe16(0); 3209 in->cmd_profile = profile; 3210 3211 mcx_cmdq_post(sc, cqe, 0); 3212 3213 error = mcx_cmdq_poll(sc, cqe, 1000); 3214 if (error != 0) { 3215 printf(", hca teardown timeout\n"); 3216 return (-1); 3217 } 3218 if (mcx_cmdq_verify(cqe) != 0) { 3219 printf(", hca teardown command corrupt\n"); 3220 return (-1); 3221 } 3222 3223 status = cqe->cq_output_data[0]; 3224 if (status != MCX_CQ_STATUS_OK) { 3225 printf(", hca teardown failed (%x)\n", status); 3226 return (-1); 3227 } 3228 3229 return (0); 3230 } 3231 3232 static int 3233 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3234 unsigned int nmb, uint64_t *ptr, uint8_t token) 3235 { 3236 caddr_t kva; 3237 uint64_t dva; 3238 int i; 3239 int error; 3240 3241 error = mcx_dmamem_alloc(sc, mxm, 3242 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3243 if (error != 0) 3244 return (error); 3245 3246 mcx_dmamem_zero(mxm); 3247 3248 dva = MCX_DMA_DVA(mxm); 3249 kva = MCX_DMA_KVA(mxm); 3250 for (i = 0; i < nmb; i++) { 3251 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3252 3253 /* patch the cqe or mbox pointing at this one */ 3254 htobem64(ptr, dva); 3255 3256 /* fill in this mbox */ 3257 htobem32(&mbox->mb_block_number, i); 3258 mbox->mb_token = token; 3259 3260 /* move to the next one */ 3261 ptr = &mbox->mb_next_ptr; 3262 3263 dva += MCX_CMDQ_MAILBOX_SIZE; 3264 kva += MCX_CMDQ_MAILBOX_SIZE; 3265 } 3266 3267 return (0); 3268 } 3269 3270 static uint32_t 3271 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3272 { 3273 uint32_t xor = 0xff; 3274 3275 /* only 3 fields get set, so mix them directly */ 3276 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3277 xor = mcx_mix_u32(xor, mb->mb_block_number); 3278 xor = mcx_mix_u8(xor, mb->mb_token); 3279 3280 return (mcx_mix_done(xor)); 3281 } 3282 3283 static void 3284 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3285 { 3286 caddr_t kva; 3287 int i; 3288 3289 kva = MCX_DMA_KVA(mxm); 3290 3291 for (i = 0; i < nmb; i++) { 3292 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3293 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3294 mb->mb_ctrl_signature = sig; 3295 mb->mb_signature = sig ^ 3296 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3297 3298 kva += MCX_CMDQ_MAILBOX_SIZE; 3299 } 3300 } 3301 3302 static void 3303 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3304 { 3305 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3306 0, MCX_DMA_LEN(mxm), ops); 3307 } 3308 3309 static struct mcx_cmdq_mailbox * 3310 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3311 { 3312 caddr_t kva; 3313 3314 kva = MCX_DMA_KVA(mxm); 3315 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3316 3317 return ((struct mcx_cmdq_mailbox *)kva); 3318 } 3319 3320 static inline void * 3321 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3322 { 3323 return (&mb->mb_data); 3324 } 3325 3326 static void 3327 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3328 void *b, size_t len) 3329 { 3330 caddr_t buf = b; 3331 struct mcx_cmdq_mailbox *mb; 3332 int i; 3333 3334 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3335 for (i = 0; i < nmb; i++) { 3336 3337 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 3338 3339 if (sizeof(mb->mb_data) >= len) 3340 break; 3341 3342 buf += sizeof(mb->mb_data); 3343 len -= sizeof(mb->mb_data); 3344 mb++; 3345 } 3346 } 3347 3348 static void 3349 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3350 struct mcx_dmamem *buf) 3351 { 3352 uint64_t *pas; 3353 int mbox, mbox_pages, i; 3354 3355 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3356 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3357 3358 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3359 pas += (offset / sizeof(*pas)); 3360 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3361 for (i = 0; i < npages; i++) { 3362 if (i == mbox_pages) { 3363 mbox++; 3364 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3365 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3366 } 3367 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3368 pas++; 3369 } 3370 } 3371 3372 static void 3373 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3374 { 3375 caddr_t buf = b; 3376 struct mcx_cmdq_mailbox *mb; 3377 int i; 3378 3379 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3380 for (i = 0; i < nmb; i++) { 3381 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 3382 3383 if (sizeof(mb->mb_data) >= len) 3384 break; 3385 3386 buf += sizeof(mb->mb_data); 3387 len -= sizeof(mb->mb_data); 3388 mb++; 3389 } 3390 } 3391 3392 static void 3393 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3394 { 3395 mcx_dmamem_free(sc, mxm); 3396 } 3397 3398 #if 0 3399 static void 3400 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3401 { 3402 unsigned int i; 3403 3404 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3405 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 3406 3407 printf(", idata "); 3408 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3409 printf("%02x", cqe->cq_input_data[i]); 3410 3411 printf(", odata "); 3412 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3413 printf("%02x", cqe->cq_output_data[i]); 3414 3415 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3416 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 3417 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3418 } 3419 3420 static void 3421 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3422 { 3423 int i, j; 3424 uint8_t *d; 3425 3426 for (i = 0; i < num; i++) { 3427 struct mcx_cmdq_mailbox *mbox; 3428 mbox = mcx_cq_mbox(mboxes, i); 3429 3430 d = mcx_cq_mbox_data(mbox); 3431 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3432 if (j != 0 && (j % 16 == 0)) 3433 printf("\n"); 3434 printf("%.2x ", d[j]); 3435 } 3436 } 3437 } 3438 #endif 3439 3440 static int 3441 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3442 int len) 3443 { 3444 struct mcx_dmamem mxm; 3445 struct mcx_cmdq_entry *cqe; 3446 struct mcx_cmd_access_reg_in *in; 3447 struct mcx_cmd_access_reg_out *out; 3448 uint8_t token = mcx_cmdq_token(sc); 3449 int error, nmb; 3450 3451 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3452 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3453 token); 3454 3455 in = mcx_cmdq_in(cqe); 3456 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3457 in->cmd_op_mod = htobe16(op); 3458 in->cmd_register_id = htobe16(reg); 3459 3460 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3461 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3462 &cqe->cq_output_ptr, token) != 0) { 3463 printf(", unable to allocate access reg mailboxen\n"); 3464 return (-1); 3465 } 3466 cqe->cq_input_ptr = cqe->cq_output_ptr; 3467 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3468 mcx_cmdq_mboxes_sign(&mxm, nmb); 3469 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3470 3471 mcx_cmdq_post(sc, cqe, 0); 3472 error = mcx_cmdq_poll(sc, cqe, 1000); 3473 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3474 3475 if (error != 0) { 3476 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3477 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3478 goto free; 3479 } 3480 error = mcx_cmdq_verify(cqe); 3481 if (error != 0) { 3482 printf("%s: access reg (%s %x) reply corrupt\n", 3483 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3484 reg); 3485 goto free; 3486 } 3487 3488 out = mcx_cmdq_out(cqe); 3489 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3490 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3491 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3492 reg, out->cmd_status, betoh32(out->cmd_syndrome)); 3493 error = -1; 3494 goto free; 3495 } 3496 3497 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3498 free: 3499 mcx_dmamem_free(sc, &mxm); 3500 3501 return (error); 3502 } 3503 3504 static int 3505 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3506 unsigned int slot) 3507 { 3508 struct mcx_cmd_set_issi_in *in; 3509 struct mcx_cmd_set_issi_out *out; 3510 uint8_t status; 3511 3512 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3513 3514 in = mcx_cmdq_in(cqe); 3515 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3516 in->cmd_op_mod = htobe16(0); 3517 in->cmd_current_issi = htobe16(MCX_ISSI); 3518 3519 mcx_cmdq_post(sc, cqe, slot); 3520 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3521 return (-1); 3522 if (mcx_cmdq_verify(cqe) != 0) 3523 return (-1); 3524 3525 status = cqe->cq_output_data[0]; 3526 if (status != MCX_CQ_STATUS_OK) 3527 return (-1); 3528 3529 return (0); 3530 } 3531 3532 static int 3533 mcx_issi(struct mcx_softc *sc) 3534 { 3535 struct mcx_dmamem mxm; 3536 struct mcx_cmdq_entry *cqe; 3537 struct mcx_cmd_query_issi_in *in; 3538 struct mcx_cmd_query_issi_il_out *out; 3539 struct mcx_cmd_query_issi_mb_out *mb; 3540 uint8_t token = mcx_cmdq_token(sc); 3541 uint8_t status; 3542 int error; 3543 3544 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3545 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3546 3547 in = mcx_cmdq_in(cqe); 3548 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3549 in->cmd_op_mod = htobe16(0); 3550 3551 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3552 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3553 &cqe->cq_output_ptr, token) != 0) { 3554 printf(", unable to allocate query issi mailbox\n"); 3555 return (-1); 3556 } 3557 mcx_cmdq_mboxes_sign(&mxm, 1); 3558 3559 mcx_cmdq_post(sc, cqe, 0); 3560 error = mcx_cmdq_poll(sc, cqe, 1000); 3561 if (error != 0) { 3562 printf(", query issi timeout\n"); 3563 goto free; 3564 } 3565 error = mcx_cmdq_verify(cqe); 3566 if (error != 0) { 3567 printf(", query issi reply corrupt\n"); 3568 goto free; 3569 } 3570 3571 status = cqe->cq_output_data[0]; 3572 switch (status) { 3573 case MCX_CQ_STATUS_OK: 3574 break; 3575 case MCX_CQ_STATUS_BAD_OPCODE: 3576 /* use ISSI 0 */ 3577 goto free; 3578 default: 3579 printf(", query issi failed (%x)\n", status); 3580 error = -1; 3581 goto free; 3582 } 3583 3584 out = mcx_cmdq_out(cqe); 3585 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3586 /* use ISSI 1 */ 3587 goto free; 3588 } 3589 3590 /* don't need to read cqe anymore, can be used for SET ISSI */ 3591 3592 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3593 CTASSERT(MCX_ISSI < NBBY); 3594 /* XXX math is hard */ 3595 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3596 /* use ISSI 0 */ 3597 goto free; 3598 } 3599 3600 if (mcx_set_issi(sc, cqe, 0) != 0) { 3601 /* ignore the error, just use ISSI 0 */ 3602 } else { 3603 /* use ISSI 1 */ 3604 } 3605 3606 free: 3607 mcx_cq_mboxes_free(sc, &mxm); 3608 return (error); 3609 } 3610 3611 static int 3612 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3613 int32_t *npages, uint16_t *func_id) 3614 { 3615 struct mcx_cmdq_entry *cqe; 3616 struct mcx_cmd_query_pages_in *in; 3617 struct mcx_cmd_query_pages_out *out; 3618 3619 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3620 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3621 3622 in = mcx_cmdq_in(cqe); 3623 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3624 in->cmd_op_mod = type; 3625 3626 mcx_cmdq_post(sc, cqe, 0); 3627 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3628 printf(", query pages timeout\n"); 3629 return (-1); 3630 } 3631 if (mcx_cmdq_verify(cqe) != 0) { 3632 printf(", query pages reply corrupt\n"); 3633 return (-1); 3634 } 3635 3636 out = mcx_cmdq_out(cqe); 3637 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3638 printf(", query pages failed (%x)\n", out->cmd_status); 3639 return (-1); 3640 } 3641 3642 *func_id = out->cmd_func_id; 3643 *npages = bemtoh32(&out->cmd_num_pages); 3644 3645 return (0); 3646 } 3647 3648 struct bus_dma_iter { 3649 bus_dmamap_t i_map; 3650 bus_size_t i_offset; 3651 unsigned int i_index; 3652 }; 3653 3654 static void 3655 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3656 { 3657 i->i_map = map; 3658 i->i_offset = 0; 3659 i->i_index = 0; 3660 } 3661 3662 static bus_addr_t 3663 bus_dma_iter_addr(struct bus_dma_iter *i) 3664 { 3665 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3666 } 3667 3668 static void 3669 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3670 { 3671 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3672 bus_size_t diff; 3673 3674 do { 3675 diff = seg->ds_len - i->i_offset; 3676 if (size < diff) 3677 break; 3678 3679 size -= diff; 3680 3681 seg++; 3682 3683 i->i_offset = 0; 3684 i->i_index++; 3685 } while (size > 0); 3686 3687 i->i_offset += size; 3688 } 3689 3690 static int 3691 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3692 { 3693 struct mcx_dmamem mxm; 3694 struct mcx_cmdq_entry *cqe; 3695 struct mcx_cmd_manage_pages_in *in; 3696 struct mcx_cmd_manage_pages_out *out; 3697 unsigned int paslen, nmb, i, j, npages; 3698 struct bus_dma_iter iter; 3699 uint64_t *pas; 3700 uint8_t status; 3701 uint8_t token = mcx_cmdq_token(sc); 3702 int error; 3703 3704 npages = mhm->mhm_npages; 3705 3706 paslen = sizeof(*pas) * npages; 3707 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3708 3709 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3710 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3711 3712 in = mcx_cmdq_in(cqe); 3713 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3714 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3715 in->cmd_func_id = func_id; 3716 htobem32(&in->cmd_input_num_entries, npages); 3717 3718 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3719 &cqe->cq_input_ptr, token) != 0) { 3720 printf(", unable to allocate manage pages mailboxen\n"); 3721 return (-1); 3722 } 3723 3724 bus_dma_iter_init(&iter, mhm->mhm_map); 3725 for (i = 0; i < nmb; i++) { 3726 unsigned int lim; 3727 3728 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3729 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3730 3731 for (j = 0; j < lim; j++) { 3732 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3733 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3734 } 3735 3736 npages -= lim; 3737 } 3738 3739 mcx_cmdq_mboxes_sign(&mxm, nmb); 3740 3741 mcx_cmdq_post(sc, cqe, 0); 3742 error = mcx_cmdq_poll(sc, cqe, 1000); 3743 if (error != 0) { 3744 printf(", manage pages timeout\n"); 3745 goto free; 3746 } 3747 error = mcx_cmdq_verify(cqe); 3748 if (error != 0) { 3749 printf(", manage pages reply corrupt\n"); 3750 goto free; 3751 } 3752 3753 status = cqe->cq_output_data[0]; 3754 if (status != MCX_CQ_STATUS_OK) { 3755 printf(", manage pages failed (%x)\n", status); 3756 error = -1; 3757 goto free; 3758 } 3759 3760 free: 3761 mcx_dmamem_free(sc, &mxm); 3762 3763 return (error); 3764 } 3765 3766 static int 3767 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3768 { 3769 int32_t npages; 3770 uint16_t func_id; 3771 3772 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3773 /* error printed by mcx_query_pages */ 3774 return (-1); 3775 } 3776 3777 if (npages < 1) 3778 return (0); 3779 3780 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3781 printf(", unable to allocate hwmem\n"); 3782 return (-1); 3783 } 3784 3785 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3786 printf(", unable to add hwmem\n"); 3787 goto free; 3788 } 3789 3790 return (0); 3791 3792 free: 3793 mcx_hwmem_free(sc, mhm); 3794 3795 return (-1); 3796 } 3797 3798 static int 3799 mcx_hca_max_caps(struct mcx_softc *sc) 3800 { 3801 struct mcx_dmamem mxm; 3802 struct mcx_cmdq_entry *cqe; 3803 struct mcx_cmd_query_hca_cap_in *in; 3804 struct mcx_cmd_query_hca_cap_out *out; 3805 struct mcx_cmdq_mailbox *mb; 3806 struct mcx_cap_device *hca; 3807 uint8_t status; 3808 uint8_t token = mcx_cmdq_token(sc); 3809 int error; 3810 3811 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3812 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3813 token); 3814 3815 in = mcx_cmdq_in(cqe); 3816 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3817 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3818 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3819 3820 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3821 &cqe->cq_output_ptr, token) != 0) { 3822 printf(", unable to allocate query hca caps mailboxen\n"); 3823 return (-1); 3824 } 3825 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3826 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3827 3828 mcx_cmdq_post(sc, cqe, 0); 3829 error = mcx_cmdq_poll(sc, cqe, 1000); 3830 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3831 3832 if (error != 0) { 3833 printf(", query hca caps timeout\n"); 3834 goto free; 3835 } 3836 error = mcx_cmdq_verify(cqe); 3837 if (error != 0) { 3838 printf(", query hca caps reply corrupt\n"); 3839 goto free; 3840 } 3841 3842 status = cqe->cq_output_data[0]; 3843 if (status != MCX_CQ_STATUS_OK) { 3844 printf(", query hca caps failed (%x)\n", status); 3845 error = -1; 3846 goto free; 3847 } 3848 3849 mb = mcx_cq_mbox(&mxm, 0); 3850 hca = mcx_cq_mbox_data(mb); 3851 3852 if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE) 3853 != MCX_CAP_DEVICE_PORT_TYPE_ETH) { 3854 printf(", not in ethernet mode\n"); 3855 error = -1; 3856 goto free; 3857 } 3858 if (hca->log_pg_sz > PAGE_SHIFT) { 3859 printf(", minimum system page shift %u is too large\n", 3860 hca->log_pg_sz); 3861 error = -1; 3862 goto free; 3863 } 3864 /* 3865 * blueflame register is split into two buffers, and we must alternate 3866 * between the two of them. 3867 */ 3868 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3869 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 3870 3871 sc->sc_mhz = bemtoh32(&hca->device_frequency_mhz); 3872 sc->sc_khz = bemtoh32(&hca->device_frequency_khz); 3873 3874 free: 3875 mcx_dmamem_free(sc, &mxm); 3876 3877 return (error); 3878 } 3879 3880 static int 3881 mcx_hca_set_caps(struct mcx_softc *sc) 3882 { 3883 struct mcx_dmamem mxm; 3884 struct mcx_cmdq_entry *cqe; 3885 struct mcx_cmd_query_hca_cap_in *in; 3886 struct mcx_cmd_query_hca_cap_out *out; 3887 struct mcx_cmdq_mailbox *mb; 3888 struct mcx_cap_device *hca; 3889 uint8_t status; 3890 uint8_t token = mcx_cmdq_token(sc); 3891 int error; 3892 3893 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3894 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3895 token); 3896 3897 in = mcx_cmdq_in(cqe); 3898 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3899 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 3900 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3901 3902 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3903 &cqe->cq_output_ptr, token) != 0) { 3904 printf(", unable to allocate manage pages mailboxen\n"); 3905 return (-1); 3906 } 3907 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3908 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3909 3910 mcx_cmdq_post(sc, cqe, 0); 3911 error = mcx_cmdq_poll(sc, cqe, 1000); 3912 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3913 3914 if (error != 0) { 3915 printf(", query hca caps timeout\n"); 3916 goto free; 3917 } 3918 error = mcx_cmdq_verify(cqe); 3919 if (error != 0) { 3920 printf(", query hca caps reply corrupt\n"); 3921 goto free; 3922 } 3923 3924 status = cqe->cq_output_data[0]; 3925 if (status != MCX_CQ_STATUS_OK) { 3926 printf(", query hca caps failed (%x)\n", status); 3927 error = -1; 3928 goto free; 3929 } 3930 3931 mb = mcx_cq_mbox(&mxm, 0); 3932 hca = mcx_cq_mbox_data(mb); 3933 3934 hca->log_pg_sz = PAGE_SHIFT; 3935 3936 free: 3937 mcx_dmamem_free(sc, &mxm); 3938 3939 return (error); 3940 } 3941 3942 3943 static int 3944 mcx_init_hca(struct mcx_softc *sc) 3945 { 3946 struct mcx_cmdq_entry *cqe; 3947 struct mcx_cmd_init_hca_in *in; 3948 struct mcx_cmd_init_hca_out *out; 3949 int error; 3950 uint8_t status; 3951 3952 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3953 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3954 3955 in = mcx_cmdq_in(cqe); 3956 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 3957 in->cmd_op_mod = htobe16(0); 3958 3959 mcx_cmdq_post(sc, cqe, 0); 3960 3961 error = mcx_cmdq_poll(sc, cqe, 1000); 3962 if (error != 0) { 3963 printf(", hca init timeout\n"); 3964 return (-1); 3965 } 3966 if (mcx_cmdq_verify(cqe) != 0) { 3967 printf(", hca init command corrupt\n"); 3968 return (-1); 3969 } 3970 3971 status = cqe->cq_output_data[0]; 3972 if (status != MCX_CQ_STATUS_OK) { 3973 printf(", hca init failed (%x)\n", status); 3974 return (-1); 3975 } 3976 3977 return (0); 3978 } 3979 3980 static int 3981 mcx_set_driver_version(struct mcx_softc *sc) 3982 { 3983 struct mcx_dmamem mxm; 3984 struct mcx_cmdq_entry *cqe; 3985 struct mcx_cmd_set_driver_version_in *in; 3986 struct mcx_cmd_set_driver_version_out *out; 3987 int error; 3988 int token; 3989 uint8_t status; 3990 3991 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3992 token = mcx_cmdq_token(sc); 3993 mcx_cmdq_init(sc, cqe, sizeof(*in) + 3994 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 3995 3996 in = mcx_cmdq_in(cqe); 3997 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 3998 in->cmd_op_mod = htobe16(0); 3999 4000 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4001 &cqe->cq_input_ptr, token) != 0) { 4002 printf(", unable to allocate set driver version mailboxen\n"); 4003 return (-1); 4004 } 4005 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 4006 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 4007 4008 mcx_cmdq_mboxes_sign(&mxm, 1); 4009 mcx_cmdq_post(sc, cqe, 0); 4010 4011 error = mcx_cmdq_poll(sc, cqe, 1000); 4012 if (error != 0) { 4013 printf(", set driver version timeout\n"); 4014 goto free; 4015 } 4016 if (mcx_cmdq_verify(cqe) != 0) { 4017 printf(", set driver version command corrupt\n"); 4018 goto free; 4019 } 4020 4021 status = cqe->cq_output_data[0]; 4022 if (status != MCX_CQ_STATUS_OK) { 4023 printf(", set driver version failed (%x)\n", status); 4024 error = -1; 4025 goto free; 4026 } 4027 4028 free: 4029 mcx_dmamem_free(sc, &mxm); 4030 4031 return (error); 4032 } 4033 4034 static int 4035 mcx_iff(struct mcx_softc *sc) 4036 { 4037 struct ifnet *ifp = &sc->sc_ac.ac_if; 4038 struct mcx_dmamem mxm; 4039 struct mcx_cmdq_entry *cqe; 4040 struct mcx_cmd_modify_nic_vport_context_in *in; 4041 struct mcx_cmd_modify_nic_vport_context_out *out; 4042 struct mcx_nic_vport_ctx *ctx; 4043 int error; 4044 int token; 4045 int insize; 4046 uint32_t dest; 4047 4048 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 4049 sc->sc_rss_flow_table_id; 4050 4051 /* enable or disable the promisc flow */ 4052 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 4053 if (sc->sc_promisc_flow_enabled == 0) { 4054 mcx_set_flow_table_entry_mac(sc, 4055 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 4056 sc->sc_promisc_flow_enabled = 1; 4057 } 4058 } else if (sc->sc_promisc_flow_enabled != 0) { 4059 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 4060 sc->sc_promisc_flow_enabled = 0; 4061 } 4062 4063 /* enable or disable the all-multicast flow */ 4064 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 4065 if (sc->sc_allmulti_flow_enabled == 0) { 4066 uint8_t mcast[ETHER_ADDR_LEN]; 4067 4068 memset(mcast, 0, sizeof(mcast)); 4069 mcast[0] = 0x01; 4070 mcx_set_flow_table_entry_mac(sc, 4071 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 4072 sc->sc_allmulti_flow_enabled = 1; 4073 } 4074 } else if (sc->sc_allmulti_flow_enabled != 0) { 4075 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 4076 sc->sc_allmulti_flow_enabled = 0; 4077 } 4078 4079 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 4080 4081 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4082 token = mcx_cmdq_token(sc); 4083 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4084 4085 in = mcx_cmdq_in(cqe); 4086 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 4087 in->cmd_op_mod = htobe16(0); 4088 in->cmd_field_select = htobe32( 4089 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 4090 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 4091 4092 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4093 printf(", unable to allocate modify " 4094 "nic vport context mailboxen\n"); 4095 return (-1); 4096 } 4097 ctx = (struct mcx_nic_vport_ctx *) 4098 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 4099 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 4100 /* 4101 * always leave promisc-all enabled on the vport since we 4102 * can't give it a vlan list, and we're already doing multicast 4103 * filtering in the flow table. 4104 */ 4105 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 4106 4107 mcx_cmdq_mboxes_sign(&mxm, 1); 4108 mcx_cmdq_post(sc, cqe, 0); 4109 4110 error = mcx_cmdq_poll(sc, cqe, 1000); 4111 if (error != 0) { 4112 printf(", modify nic vport context timeout\n"); 4113 goto free; 4114 } 4115 if (mcx_cmdq_verify(cqe) != 0) { 4116 printf(", modify nic vport context command corrupt\n"); 4117 goto free; 4118 } 4119 4120 out = mcx_cmdq_out(cqe); 4121 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4122 printf(", modify nic vport context failed (%x, %x)\n", 4123 out->cmd_status, betoh32(out->cmd_syndrome)); 4124 error = -1; 4125 goto free; 4126 } 4127 4128 free: 4129 mcx_dmamem_free(sc, &mxm); 4130 4131 return (error); 4132 } 4133 4134 static int 4135 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 4136 { 4137 struct mcx_cmdq_entry *cqe; 4138 struct mcx_cmd_alloc_uar_in *in; 4139 struct mcx_cmd_alloc_uar_out *out; 4140 int error; 4141 4142 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4143 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4144 4145 in = mcx_cmdq_in(cqe); 4146 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 4147 in->cmd_op_mod = htobe16(0); 4148 4149 mcx_cmdq_post(sc, cqe, 0); 4150 4151 error = mcx_cmdq_poll(sc, cqe, 1000); 4152 if (error != 0) { 4153 printf(", alloc uar timeout\n"); 4154 return (-1); 4155 } 4156 if (mcx_cmdq_verify(cqe) != 0) { 4157 printf(", alloc uar command corrupt\n"); 4158 return (-1); 4159 } 4160 4161 out = mcx_cmdq_out(cqe); 4162 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4163 printf(", alloc uar failed (%x)\n", out->cmd_status); 4164 return (-1); 4165 } 4166 4167 *uar = mcx_get_id(out->cmd_uar); 4168 return (0); 4169 } 4170 4171 static int 4172 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4173 uint64_t events, int vector) 4174 { 4175 struct mcx_cmdq_entry *cqe; 4176 struct mcx_dmamem mxm; 4177 struct mcx_cmd_create_eq_in *in; 4178 struct mcx_cmd_create_eq_mb_in *mbin; 4179 struct mcx_cmd_create_eq_out *out; 4180 struct mcx_eq_entry *eqe; 4181 int error; 4182 uint64_t *pas; 4183 int insize, npages, paslen, i, token; 4184 4185 eq->eq_cons = 0; 4186 4187 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4188 MCX_PAGE_SIZE); 4189 paslen = npages * sizeof(*pas); 4190 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4191 4192 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4193 MCX_PAGE_SIZE) != 0) { 4194 printf(", unable to allocate event queue memory\n"); 4195 return (-1); 4196 } 4197 4198 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4199 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4200 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4201 } 4202 4203 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4204 token = mcx_cmdq_token(sc); 4205 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4206 4207 in = mcx_cmdq_in(cqe); 4208 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4209 in->cmd_op_mod = htobe16(0); 4210 4211 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4212 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4213 &cqe->cq_input_ptr, token) != 0) { 4214 printf(", unable to allocate create eq mailboxen\n"); 4215 goto free_eq; 4216 } 4217 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4218 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4219 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4220 mbin->cmd_eq_ctx.eq_intr = vector; 4221 mbin->cmd_event_bitmask = htobe64(events); 4222 4223 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4224 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 4225 4226 /* physical addresses follow the mailbox in data */ 4227 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4228 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4229 mcx_cmdq_post(sc, cqe, 0); 4230 4231 error = mcx_cmdq_poll(sc, cqe, 1000); 4232 if (error != 0) { 4233 printf(", create eq timeout\n"); 4234 goto free_mxm; 4235 } 4236 if (mcx_cmdq_verify(cqe) != 0) { 4237 printf(", create eq command corrupt\n"); 4238 goto free_mxm; 4239 } 4240 4241 out = mcx_cmdq_out(cqe); 4242 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4243 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4244 betoh32(out->cmd_syndrome)); 4245 goto free_mxm; 4246 } 4247 4248 eq->eq_n = mcx_get_id(out->cmd_eqn); 4249 4250 mcx_dmamem_free(sc, &mxm); 4251 4252 mcx_arm_eq(sc, eq, uar); 4253 4254 return (0); 4255 4256 free_mxm: 4257 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4258 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 4259 mcx_dmamem_free(sc, &mxm); 4260 free_eq: 4261 mcx_dmamem_free(sc, &eq->eq_mem); 4262 return (-1); 4263 } 4264 4265 static int 4266 mcx_alloc_pd(struct mcx_softc *sc) 4267 { 4268 struct mcx_cmdq_entry *cqe; 4269 struct mcx_cmd_alloc_pd_in *in; 4270 struct mcx_cmd_alloc_pd_out *out; 4271 int error; 4272 4273 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4274 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4275 4276 in = mcx_cmdq_in(cqe); 4277 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4278 in->cmd_op_mod = htobe16(0); 4279 4280 mcx_cmdq_post(sc, cqe, 0); 4281 4282 error = mcx_cmdq_poll(sc, cqe, 1000); 4283 if (error != 0) { 4284 printf(", alloc pd timeout\n"); 4285 return (-1); 4286 } 4287 if (mcx_cmdq_verify(cqe) != 0) { 4288 printf(", alloc pd command corrupt\n"); 4289 return (-1); 4290 } 4291 4292 out = mcx_cmdq_out(cqe); 4293 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4294 printf(", alloc pd failed (%x)\n", out->cmd_status); 4295 return (-1); 4296 } 4297 4298 sc->sc_pd = mcx_get_id(out->cmd_pd); 4299 return (0); 4300 } 4301 4302 static int 4303 mcx_alloc_tdomain(struct mcx_softc *sc) 4304 { 4305 struct mcx_cmdq_entry *cqe; 4306 struct mcx_cmd_alloc_td_in *in; 4307 struct mcx_cmd_alloc_td_out *out; 4308 int error; 4309 4310 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4311 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4312 4313 in = mcx_cmdq_in(cqe); 4314 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4315 in->cmd_op_mod = htobe16(0); 4316 4317 mcx_cmdq_post(sc, cqe, 0); 4318 4319 error = mcx_cmdq_poll(sc, cqe, 1000); 4320 if (error != 0) { 4321 printf(", alloc transport domain timeout\n"); 4322 return (-1); 4323 } 4324 if (mcx_cmdq_verify(cqe) != 0) { 4325 printf(", alloc transport domain command corrupt\n"); 4326 return (-1); 4327 } 4328 4329 out = mcx_cmdq_out(cqe); 4330 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4331 printf(", alloc transport domain failed (%x)\n", 4332 out->cmd_status); 4333 return (-1); 4334 } 4335 4336 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4337 return (0); 4338 } 4339 4340 static int 4341 mcx_query_nic_vport_context(struct mcx_softc *sc) 4342 { 4343 struct mcx_dmamem mxm; 4344 struct mcx_cmdq_entry *cqe; 4345 struct mcx_cmd_query_nic_vport_context_in *in; 4346 struct mcx_cmd_query_nic_vport_context_out *out; 4347 struct mcx_nic_vport_ctx *ctx; 4348 uint8_t *addr; 4349 int error, token, i; 4350 4351 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4352 token = mcx_cmdq_token(sc); 4353 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4354 4355 in = mcx_cmdq_in(cqe); 4356 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4357 in->cmd_op_mod = htobe16(0); 4358 in->cmd_allowed_list_type = 0; 4359 4360 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4361 &cqe->cq_output_ptr, token) != 0) { 4362 printf(", unable to allocate " 4363 "query nic vport context mailboxen\n"); 4364 return (-1); 4365 } 4366 mcx_cmdq_mboxes_sign(&mxm, 1); 4367 mcx_cmdq_post(sc, cqe, 0); 4368 4369 error = mcx_cmdq_poll(sc, cqe, 1000); 4370 if (error != 0) { 4371 printf(", query nic vport context timeout\n"); 4372 goto free; 4373 } 4374 if (mcx_cmdq_verify(cqe) != 0) { 4375 printf(", query nic vport context command corrupt\n"); 4376 goto free; 4377 } 4378 4379 out = mcx_cmdq_out(cqe); 4380 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4381 printf(", query nic vport context failed (%x, %x)\n", 4382 out->cmd_status, betoh32(out->cmd_syndrome)); 4383 error = -1; 4384 goto free; 4385 } 4386 4387 ctx = (struct mcx_nic_vport_ctx *) 4388 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4389 addr = (uint8_t *)&ctx->vp_perm_addr; 4390 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4391 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 4392 } 4393 free: 4394 mcx_dmamem_free(sc, &mxm); 4395 4396 return (error); 4397 } 4398 4399 static int 4400 mcx_query_special_contexts(struct mcx_softc *sc) 4401 { 4402 struct mcx_cmdq_entry *cqe; 4403 struct mcx_cmd_query_special_ctx_in *in; 4404 struct mcx_cmd_query_special_ctx_out *out; 4405 int error; 4406 4407 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4408 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4409 4410 in = mcx_cmdq_in(cqe); 4411 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4412 in->cmd_op_mod = htobe16(0); 4413 4414 mcx_cmdq_post(sc, cqe, 0); 4415 4416 error = mcx_cmdq_poll(sc, cqe, 1000); 4417 if (error != 0) { 4418 printf(", query special contexts timeout\n"); 4419 return (-1); 4420 } 4421 if (mcx_cmdq_verify(cqe) != 0) { 4422 printf(", query special contexts command corrupt\n"); 4423 return (-1); 4424 } 4425 4426 out = mcx_cmdq_out(cqe); 4427 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4428 printf(", query special contexts failed (%x)\n", 4429 out->cmd_status); 4430 return (-1); 4431 } 4432 4433 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 4434 return (0); 4435 } 4436 4437 static int 4438 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4439 { 4440 struct mcx_reg_pmtu pmtu; 4441 int error; 4442 4443 /* read max mtu */ 4444 memset(&pmtu, 0, sizeof(pmtu)); 4445 pmtu.rp_local_port = 1; 4446 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4447 sizeof(pmtu)); 4448 if (error != 0) { 4449 printf(", unable to get port MTU\n"); 4450 return error; 4451 } 4452 4453 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 4454 pmtu.rp_admin_mtu = htobe16(mtu); 4455 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4456 sizeof(pmtu)); 4457 if (error != 0) { 4458 printf(", unable to set port MTU\n"); 4459 return error; 4460 } 4461 4462 sc->sc_hardmtu = mtu; 4463 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4464 return 0; 4465 } 4466 4467 static int 4468 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4469 { 4470 struct mcx_cmdq_entry *cmde; 4471 struct mcx_cq_entry *cqe; 4472 struct mcx_dmamem mxm; 4473 struct mcx_cmd_create_cq_in *in; 4474 struct mcx_cmd_create_cq_mb_in *mbin; 4475 struct mcx_cmd_create_cq_out *out; 4476 int error; 4477 uint64_t *pas; 4478 int insize, npages, paslen, i, token; 4479 4480 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4481 MCX_PAGE_SIZE); 4482 paslen = npages * sizeof(*pas); 4483 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4484 4485 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4486 MCX_PAGE_SIZE) != 0) { 4487 printf("%s: unable to allocate completion queue memory\n", 4488 DEVNAME(sc)); 4489 return (-1); 4490 } 4491 cqe = MCX_DMA_KVA(&cq->cq_mem); 4492 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4493 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4494 } 4495 4496 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4497 token = mcx_cmdq_token(sc); 4498 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4499 4500 in = mcx_cmdq_in(cmde); 4501 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4502 in->cmd_op_mod = htobe16(0); 4503 4504 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4505 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4506 &cmde->cq_input_ptr, token) != 0) { 4507 printf("%s: unable to allocate create cq mailboxen\n", 4508 DEVNAME(sc)); 4509 goto free_cq; 4510 } 4511 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4512 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4513 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4514 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4515 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4516 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4517 MCX_CQ_MOD_COUNTER); 4518 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4519 MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4520 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db)); 4521 4522 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4523 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 4524 4525 /* physical addresses follow the mailbox in data */ 4526 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4527 mcx_cmdq_post(sc, cmde, 0); 4528 4529 error = mcx_cmdq_poll(sc, cmde, 1000); 4530 if (error != 0) { 4531 printf("%s: create cq timeout\n", DEVNAME(sc)); 4532 goto free_mxm; 4533 } 4534 if (mcx_cmdq_verify(cmde) != 0) { 4535 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4536 goto free_mxm; 4537 } 4538 4539 out = mcx_cmdq_out(cmde); 4540 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4541 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4542 out->cmd_status, betoh32(out->cmd_syndrome)); 4543 goto free_mxm; 4544 } 4545 4546 cq->cq_n = mcx_get_id(out->cmd_cqn); 4547 cq->cq_cons = 0; 4548 cq->cq_count = 0; 4549 cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + 4550 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4551 4552 mcx_dmamem_free(sc, &mxm); 4553 4554 mcx_arm_cq(sc, cq, uar); 4555 4556 return (0); 4557 4558 free_mxm: 4559 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4560 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4561 mcx_dmamem_free(sc, &mxm); 4562 free_cq: 4563 mcx_dmamem_free(sc, &cq->cq_mem); 4564 return (-1); 4565 } 4566 4567 static int 4568 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4569 { 4570 struct mcx_cmdq_entry *cqe; 4571 struct mcx_cmd_destroy_cq_in *in; 4572 struct mcx_cmd_destroy_cq_out *out; 4573 int error; 4574 int token; 4575 4576 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4577 token = mcx_cmdq_token(sc); 4578 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4579 4580 in = mcx_cmdq_in(cqe); 4581 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4582 in->cmd_op_mod = htobe16(0); 4583 in->cmd_cqn = htobe32(cq->cq_n); 4584 4585 mcx_cmdq_post(sc, cqe, 0); 4586 error = mcx_cmdq_poll(sc, cqe, 1000); 4587 if (error != 0) { 4588 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4589 return error; 4590 } 4591 if (mcx_cmdq_verify(cqe) != 0) { 4592 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4593 return error; 4594 } 4595 4596 out = mcx_cmdq_out(cqe); 4597 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4598 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4599 out->cmd_status, betoh32(out->cmd_syndrome)); 4600 return -1; 4601 } 4602 4603 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4604 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4605 mcx_dmamem_free(sc, &cq->cq_mem); 4606 4607 cq->cq_n = 0; 4608 cq->cq_cons = 0; 4609 cq->cq_count = 0; 4610 return 0; 4611 } 4612 4613 static int 4614 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4615 { 4616 struct mcx_cmdq_entry *cqe; 4617 struct mcx_dmamem mxm; 4618 struct mcx_cmd_create_rq_in *in; 4619 struct mcx_cmd_create_rq_out *out; 4620 struct mcx_rq_ctx *mbin; 4621 int error; 4622 uint64_t *pas; 4623 uint32_t rq_flags; 4624 uint8_t *doorbell; 4625 int insize, npages, paslen, token; 4626 4627 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4628 MCX_PAGE_SIZE); 4629 paslen = npages * sizeof(*pas); 4630 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4631 4632 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4633 MCX_PAGE_SIZE) != 0) { 4634 printf("%s: unable to allocate receive queue memory\n", 4635 DEVNAME(sc)); 4636 return (-1); 4637 } 4638 4639 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4640 token = mcx_cmdq_token(sc); 4641 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4642 4643 in = mcx_cmdq_in(cqe); 4644 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4645 in->cmd_op_mod = htobe16(0); 4646 4647 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4648 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4649 &cqe->cq_input_ptr, token) != 0) { 4650 printf("%s: unable to allocate create rq mailboxen\n", 4651 DEVNAME(sc)); 4652 goto free_rq; 4653 } 4654 mbin = (struct mcx_rq_ctx *) 4655 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4656 rq_flags = MCX_RQ_CTX_RLKEY; 4657 #if NVLAN == 0 4658 rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS; 4659 #endif 4660 mbin->rq_flags = htobe32(rq_flags); 4661 mbin->rq_cqn = htobe32(cqn); 4662 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4663 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4664 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4665 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 4666 mbin->rq_wq.wq_log_stride = htobe16(4); 4667 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4668 4669 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4670 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 4671 4672 /* physical addresses follow the mailbox in data */ 4673 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4674 mcx_cmdq_post(sc, cqe, 0); 4675 4676 error = mcx_cmdq_poll(sc, cqe, 1000); 4677 if (error != 0) { 4678 printf("%s: create rq timeout\n", DEVNAME(sc)); 4679 goto free_mxm; 4680 } 4681 if (mcx_cmdq_verify(cqe) != 0) { 4682 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4683 goto free_mxm; 4684 } 4685 4686 out = mcx_cmdq_out(cqe); 4687 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4688 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4689 out->cmd_status, betoh32(out->cmd_syndrome)); 4690 goto free_mxm; 4691 } 4692 4693 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4694 4695 mcx_dmamem_free(sc, &mxm); 4696 4697 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4698 rx->rx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 4699 (db * MCX_WQ_DOORBELL_STRIDE)); 4700 4701 return (0); 4702 4703 free_mxm: 4704 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4705 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4706 mcx_dmamem_free(sc, &mxm); 4707 free_rq: 4708 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4709 return (-1); 4710 } 4711 4712 static int 4713 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4714 { 4715 struct mcx_cmdq_entry *cqe; 4716 struct mcx_dmamem mxm; 4717 struct mcx_cmd_modify_rq_in *in; 4718 struct mcx_cmd_modify_rq_mb_in *mbin; 4719 struct mcx_cmd_modify_rq_out *out; 4720 int error; 4721 int token; 4722 4723 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4724 token = mcx_cmdq_token(sc); 4725 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4726 sizeof(*out), token); 4727 4728 in = mcx_cmdq_in(cqe); 4729 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4730 in->cmd_op_mod = htobe16(0); 4731 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4732 4733 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4734 &cqe->cq_input_ptr, token) != 0) { 4735 printf("%s: unable to allocate modify rq mailbox\n", 4736 DEVNAME(sc)); 4737 return (-1); 4738 } 4739 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4740 mbin->cmd_rq_ctx.rq_flags = htobe32( 4741 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4742 4743 mcx_cmdq_mboxes_sign(&mxm, 1); 4744 mcx_cmdq_post(sc, cqe, 0); 4745 error = mcx_cmdq_poll(sc, cqe, 1000); 4746 if (error != 0) { 4747 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4748 goto free; 4749 } 4750 if (mcx_cmdq_verify(cqe) != 0) { 4751 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4752 goto free; 4753 } 4754 4755 out = mcx_cmdq_out(cqe); 4756 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4757 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4758 out->cmd_status, betoh32(out->cmd_syndrome)); 4759 error = -1; 4760 goto free; 4761 } 4762 4763 free: 4764 mcx_dmamem_free(sc, &mxm); 4765 return (error); 4766 } 4767 4768 static int 4769 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4770 { 4771 struct mcx_cmdq_entry *cqe; 4772 struct mcx_cmd_destroy_rq_in *in; 4773 struct mcx_cmd_destroy_rq_out *out; 4774 int error; 4775 int token; 4776 4777 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4778 token = mcx_cmdq_token(sc); 4779 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4780 4781 in = mcx_cmdq_in(cqe); 4782 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4783 in->cmd_op_mod = htobe16(0); 4784 in->cmd_rqn = htobe32(rx->rx_rqn); 4785 4786 mcx_cmdq_post(sc, cqe, 0); 4787 error = mcx_cmdq_poll(sc, cqe, 1000); 4788 if (error != 0) { 4789 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4790 return error; 4791 } 4792 if (mcx_cmdq_verify(cqe) != 0) { 4793 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4794 return error; 4795 } 4796 4797 out = mcx_cmdq_out(cqe); 4798 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4799 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4800 out->cmd_status, betoh32(out->cmd_syndrome)); 4801 return -1; 4802 } 4803 4804 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4805 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4806 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4807 4808 rx->rx_rqn = 0; 4809 return 0; 4810 } 4811 4812 static int 4813 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 4814 { 4815 struct mcx_cmdq_entry *cqe; 4816 struct mcx_dmamem mxm; 4817 struct mcx_cmd_create_tir_in *in; 4818 struct mcx_cmd_create_tir_mb_in *mbin; 4819 struct mcx_cmd_create_tir_out *out; 4820 int error; 4821 int token; 4822 4823 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4824 token = mcx_cmdq_token(sc); 4825 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4826 sizeof(*out), token); 4827 4828 in = mcx_cmdq_in(cqe); 4829 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4830 in->cmd_op_mod = htobe16(0); 4831 4832 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4833 &cqe->cq_input_ptr, token) != 0) { 4834 printf("%s: unable to allocate create tir mailbox\n", 4835 DEVNAME(sc)); 4836 return (-1); 4837 } 4838 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4839 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4840 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 4841 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4842 4843 mcx_cmdq_post(sc, cqe, 0); 4844 error = mcx_cmdq_poll(sc, cqe, 1000); 4845 if (error != 0) { 4846 printf("%s: create tir timeout\n", DEVNAME(sc)); 4847 goto free; 4848 } 4849 if (mcx_cmdq_verify(cqe) != 0) { 4850 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4851 goto free; 4852 } 4853 4854 out = mcx_cmdq_out(cqe); 4855 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4856 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4857 out->cmd_status, betoh32(out->cmd_syndrome)); 4858 error = -1; 4859 goto free; 4860 } 4861 4862 *tirn = mcx_get_id(out->cmd_tirn); 4863 free: 4864 mcx_dmamem_free(sc, &mxm); 4865 return (error); 4866 } 4867 4868 static int 4869 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 4870 int *tirn) 4871 { 4872 struct mcx_cmdq_entry *cqe; 4873 struct mcx_dmamem mxm; 4874 struct mcx_cmd_create_tir_in *in; 4875 struct mcx_cmd_create_tir_mb_in *mbin; 4876 struct mcx_cmd_create_tir_out *out; 4877 int error; 4878 int token; 4879 4880 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4881 token = mcx_cmdq_token(sc); 4882 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4883 sizeof(*out), token); 4884 4885 in = mcx_cmdq_in(cqe); 4886 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4887 in->cmd_op_mod = htobe16(0); 4888 4889 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4890 &cqe->cq_input_ptr, token) != 0) { 4891 printf("%s: unable to allocate create tir mailbox\n", 4892 DEVNAME(sc)); 4893 return (-1); 4894 } 4895 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4896 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 4897 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 4898 mbin->cmd_indir_table = htobe32(rqtn); 4899 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 4900 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 4901 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 4902 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 4903 sizeof(mbin->cmd_rx_hash_key)); 4904 4905 mcx_cmdq_post(sc, cqe, 0); 4906 error = mcx_cmdq_poll(sc, cqe, 1000); 4907 if (error != 0) { 4908 printf("%s: create tir timeout\n", DEVNAME(sc)); 4909 goto free; 4910 } 4911 if (mcx_cmdq_verify(cqe) != 0) { 4912 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4913 goto free; 4914 } 4915 4916 out = mcx_cmdq_out(cqe); 4917 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4918 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4919 out->cmd_status, betoh32(out->cmd_syndrome)); 4920 error = -1; 4921 goto free; 4922 } 4923 4924 *tirn = mcx_get_id(out->cmd_tirn); 4925 free: 4926 mcx_dmamem_free(sc, &mxm); 4927 return (error); 4928 } 4929 4930 static int 4931 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 4932 { 4933 struct mcx_cmdq_entry *cqe; 4934 struct mcx_cmd_destroy_tir_in *in; 4935 struct mcx_cmd_destroy_tir_out *out; 4936 int error; 4937 int token; 4938 4939 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4940 token = mcx_cmdq_token(sc); 4941 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4942 4943 in = mcx_cmdq_in(cqe); 4944 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 4945 in->cmd_op_mod = htobe16(0); 4946 in->cmd_tirn = htobe32(tirn); 4947 4948 mcx_cmdq_post(sc, cqe, 0); 4949 error = mcx_cmdq_poll(sc, cqe, 1000); 4950 if (error != 0) { 4951 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 4952 return error; 4953 } 4954 if (mcx_cmdq_verify(cqe) != 0) { 4955 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 4956 return error; 4957 } 4958 4959 out = mcx_cmdq_out(cqe); 4960 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4961 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 4962 out->cmd_status, betoh32(out->cmd_syndrome)); 4963 return -1; 4964 } 4965 4966 return (0); 4967 } 4968 4969 static int 4970 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 4971 int cqn) 4972 { 4973 struct mcx_cmdq_entry *cqe; 4974 struct mcx_dmamem mxm; 4975 struct mcx_cmd_create_sq_in *in; 4976 struct mcx_sq_ctx *mbin; 4977 struct mcx_cmd_create_sq_out *out; 4978 int error; 4979 uint64_t *pas; 4980 uint8_t *doorbell; 4981 int insize, npages, paslen, token; 4982 4983 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 4984 MCX_PAGE_SIZE); 4985 paslen = npages * sizeof(*pas); 4986 insize = sizeof(struct mcx_sq_ctx) + paslen; 4987 4988 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 4989 MCX_PAGE_SIZE) != 0) { 4990 printf("%s: unable to allocate send queue memory\n", 4991 DEVNAME(sc)); 4992 return (-1); 4993 } 4994 4995 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4996 token = mcx_cmdq_token(sc); 4997 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 4998 token); 4999 5000 in = mcx_cmdq_in(cqe); 5001 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 5002 in->cmd_op_mod = htobe16(0); 5003 5004 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 5005 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 5006 &cqe->cq_input_ptr, token) != 0) { 5007 printf("%s: unable to allocate create sq mailboxen\n", 5008 DEVNAME(sc)); 5009 goto free_sq; 5010 } 5011 mbin = (struct mcx_sq_ctx *) 5012 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 5013 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 5014 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 5015 mbin->sq_cqn = htobe32(cqn); 5016 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 5017 mbin->sq_tis_num = htobe32(sc->sc_tis); 5018 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 5019 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 5020 mbin->sq_wq.wq_uar_page = htobe32(uar); 5021 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 5022 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 5023 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 5024 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 5025 5026 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5027 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 5028 5029 /* physical addresses follow the mailbox in data */ 5030 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 5031 npages, &tx->tx_sq_mem); 5032 mcx_cmdq_post(sc, cqe, 0); 5033 5034 error = mcx_cmdq_poll(sc, cqe, 1000); 5035 if (error != 0) { 5036 printf("%s: create sq timeout\n", DEVNAME(sc)); 5037 goto free_mxm; 5038 } 5039 if (mcx_cmdq_verify(cqe) != 0) { 5040 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 5041 goto free_mxm; 5042 } 5043 5044 out = mcx_cmdq_out(cqe); 5045 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5046 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 5047 out->cmd_status, betoh32(out->cmd_syndrome)); 5048 goto free_mxm; 5049 } 5050 5051 tx->tx_uar = uar; 5052 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 5053 5054 mcx_dmamem_free(sc, &mxm); 5055 5056 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 5057 tx->tx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 5058 (db * MCX_WQ_DOORBELL_STRIDE) + 4); 5059 5060 return (0); 5061 5062 free_mxm: 5063 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5064 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5065 mcx_dmamem_free(sc, &mxm); 5066 free_sq: 5067 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5068 return (-1); 5069 } 5070 5071 static int 5072 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5073 { 5074 struct mcx_cmdq_entry *cqe; 5075 struct mcx_cmd_destroy_sq_in *in; 5076 struct mcx_cmd_destroy_sq_out *out; 5077 int error; 5078 int token; 5079 5080 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5081 token = mcx_cmdq_token(sc); 5082 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5083 5084 in = mcx_cmdq_in(cqe); 5085 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 5086 in->cmd_op_mod = htobe16(0); 5087 in->cmd_sqn = htobe32(tx->tx_sqn); 5088 5089 mcx_cmdq_post(sc, cqe, 0); 5090 error = mcx_cmdq_poll(sc, cqe, 1000); 5091 if (error != 0) { 5092 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 5093 return error; 5094 } 5095 if (mcx_cmdq_verify(cqe) != 0) { 5096 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 5097 return error; 5098 } 5099 5100 out = mcx_cmdq_out(cqe); 5101 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5102 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 5103 out->cmd_status, betoh32(out->cmd_syndrome)); 5104 return -1; 5105 } 5106 5107 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5108 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5109 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5110 5111 tx->tx_sqn = 0; 5112 return 0; 5113 } 5114 5115 static int 5116 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5117 { 5118 struct mcx_cmdq_entry *cqe; 5119 struct mcx_dmamem mxm; 5120 struct mcx_cmd_modify_sq_in *in; 5121 struct mcx_cmd_modify_sq_mb_in *mbin; 5122 struct mcx_cmd_modify_sq_out *out; 5123 int error; 5124 int token; 5125 5126 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5127 token = mcx_cmdq_token(sc); 5128 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5129 sizeof(*out), token); 5130 5131 in = mcx_cmdq_in(cqe); 5132 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 5133 in->cmd_op_mod = htobe16(0); 5134 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 5135 5136 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5137 &cqe->cq_input_ptr, token) != 0) { 5138 printf("%s: unable to allocate modify sq mailbox\n", 5139 DEVNAME(sc)); 5140 return (-1); 5141 } 5142 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5143 mbin->cmd_sq_ctx.sq_flags = htobe32( 5144 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 5145 5146 mcx_cmdq_mboxes_sign(&mxm, 1); 5147 mcx_cmdq_post(sc, cqe, 0); 5148 error = mcx_cmdq_poll(sc, cqe, 1000); 5149 if (error != 0) { 5150 printf("%s: modify sq timeout\n", DEVNAME(sc)); 5151 goto free; 5152 } 5153 if (mcx_cmdq_verify(cqe) != 0) { 5154 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 5155 goto free; 5156 } 5157 5158 out = mcx_cmdq_out(cqe); 5159 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5160 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 5161 out->cmd_status, betoh32(out->cmd_syndrome)); 5162 error = -1; 5163 goto free; 5164 } 5165 5166 free: 5167 mcx_dmamem_free(sc, &mxm); 5168 return (error); 5169 } 5170 5171 static int 5172 mcx_create_tis(struct mcx_softc *sc, int *tis) 5173 { 5174 struct mcx_cmdq_entry *cqe; 5175 struct mcx_dmamem mxm; 5176 struct mcx_cmd_create_tis_in *in; 5177 struct mcx_cmd_create_tis_mb_in *mbin; 5178 struct mcx_cmd_create_tis_out *out; 5179 int error; 5180 int token; 5181 5182 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5183 token = mcx_cmdq_token(sc); 5184 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5185 sizeof(*out), token); 5186 5187 in = mcx_cmdq_in(cqe); 5188 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 5189 in->cmd_op_mod = htobe16(0); 5190 5191 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5192 &cqe->cq_input_ptr, token) != 0) { 5193 printf("%s: unable to allocate create tis mailbox\n", 5194 DEVNAME(sc)); 5195 return (-1); 5196 } 5197 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5198 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5199 5200 mcx_cmdq_mboxes_sign(&mxm, 1); 5201 mcx_cmdq_post(sc, cqe, 0); 5202 error = mcx_cmdq_poll(sc, cqe, 1000); 5203 if (error != 0) { 5204 printf("%s: create tis timeout\n", DEVNAME(sc)); 5205 goto free; 5206 } 5207 if (mcx_cmdq_verify(cqe) != 0) { 5208 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5209 goto free; 5210 } 5211 5212 out = mcx_cmdq_out(cqe); 5213 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5214 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5215 out->cmd_status, betoh32(out->cmd_syndrome)); 5216 error = -1; 5217 goto free; 5218 } 5219 5220 *tis = mcx_get_id(out->cmd_tisn); 5221 free: 5222 mcx_dmamem_free(sc, &mxm); 5223 return (error); 5224 } 5225 5226 static int 5227 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5228 { 5229 struct mcx_cmdq_entry *cqe; 5230 struct mcx_cmd_destroy_tis_in *in; 5231 struct mcx_cmd_destroy_tis_out *out; 5232 int error; 5233 int token; 5234 5235 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5236 token = mcx_cmdq_token(sc); 5237 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5238 5239 in = mcx_cmdq_in(cqe); 5240 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5241 in->cmd_op_mod = htobe16(0); 5242 in->cmd_tisn = htobe32(tis); 5243 5244 mcx_cmdq_post(sc, cqe, 0); 5245 error = mcx_cmdq_poll(sc, cqe, 1000); 5246 if (error != 0) { 5247 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5248 return error; 5249 } 5250 if (mcx_cmdq_verify(cqe) != 0) { 5251 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5252 return error; 5253 } 5254 5255 out = mcx_cmdq_out(cqe); 5256 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5257 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5258 out->cmd_status, betoh32(out->cmd_syndrome)); 5259 return -1; 5260 } 5261 5262 return 0; 5263 } 5264 5265 static int 5266 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5267 { 5268 struct mcx_cmdq_entry *cqe; 5269 struct mcx_dmamem mxm; 5270 struct mcx_cmd_create_rqt_in *in; 5271 struct mcx_cmd_create_rqt_mb_in *mbin; 5272 struct mcx_cmd_create_rqt_out *out; 5273 struct mcx_rqt_ctx *rqt_ctx; 5274 int *rqtn; 5275 int error; 5276 int token; 5277 int i; 5278 5279 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5280 token = mcx_cmdq_token(sc); 5281 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5282 (size * sizeof(int)), sizeof(*out), token); 5283 5284 in = mcx_cmdq_in(cqe); 5285 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5286 in->cmd_op_mod = htobe16(0); 5287 5288 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5289 &cqe->cq_input_ptr, token) != 0) { 5290 printf("%s: unable to allocate create rqt mailbox\n", 5291 DEVNAME(sc)); 5292 return (-1); 5293 } 5294 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5295 rqt_ctx = &mbin->cmd_rqt; 5296 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5297 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5298 5299 /* rqt list follows the rqt context */ 5300 rqtn = (int *)(rqt_ctx + 1); 5301 for (i = 0; i < size; i++) { 5302 rqtn[i] = htobe32(rqns[i]); 5303 } 5304 5305 mcx_cmdq_mboxes_sign(&mxm, 1); 5306 mcx_cmdq_post(sc, cqe, 0); 5307 error = mcx_cmdq_poll(sc, cqe, 1000); 5308 if (error != 0) { 5309 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5310 goto free; 5311 } 5312 if (mcx_cmdq_verify(cqe) != 0) { 5313 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5314 goto free; 5315 } 5316 5317 out = mcx_cmdq_out(cqe); 5318 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5319 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5320 out->cmd_status, betoh32(out->cmd_syndrome)); 5321 error = -1; 5322 goto free; 5323 } 5324 5325 *rqt = mcx_get_id(out->cmd_rqtn); 5326 return (0); 5327 free: 5328 mcx_dmamem_free(sc, &mxm); 5329 return (error); 5330 } 5331 5332 static int 5333 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5334 { 5335 struct mcx_cmdq_entry *cqe; 5336 struct mcx_cmd_destroy_rqt_in *in; 5337 struct mcx_cmd_destroy_rqt_out *out; 5338 int error; 5339 int token; 5340 5341 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5342 token = mcx_cmdq_token(sc); 5343 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5344 5345 in = mcx_cmdq_in(cqe); 5346 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5347 in->cmd_op_mod = htobe16(0); 5348 in->cmd_rqtn = htobe32(rqt); 5349 5350 mcx_cmdq_post(sc, cqe, 0); 5351 error = mcx_cmdq_poll(sc, cqe, 1000); 5352 if (error != 0) { 5353 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5354 return error; 5355 } 5356 if (mcx_cmdq_verify(cqe) != 0) { 5357 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5358 return error; 5359 } 5360 5361 out = mcx_cmdq_out(cqe); 5362 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5363 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5364 out->cmd_status, betoh32(out->cmd_syndrome)); 5365 return -1; 5366 } 5367 5368 return 0; 5369 } 5370 5371 #if 0 5372 static int 5373 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5374 { 5375 struct mcx_cmdq_entry *cqe; 5376 struct mcx_cmd_alloc_flow_counter_in *in; 5377 struct mcx_cmd_alloc_flow_counter_out *out; 5378 int error; 5379 5380 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5381 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5382 5383 in = mcx_cmdq_in(cqe); 5384 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5385 in->cmd_op_mod = htobe16(0); 5386 5387 mcx_cmdq_post(sc, cqe, 0); 5388 5389 error = mcx_cmdq_poll(sc, cqe, 1000); 5390 if (error != 0) { 5391 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5392 return (-1); 5393 } 5394 if (mcx_cmdq_verify(cqe) != 0) { 5395 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5396 return (-1); 5397 } 5398 5399 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5400 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5401 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5402 out->cmd_status); 5403 return (-1); 5404 } 5405 5406 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 5407 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5408 5409 return (0); 5410 } 5411 #endif 5412 5413 static int 5414 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5415 int *flow_table_id) 5416 { 5417 struct mcx_cmdq_entry *cqe; 5418 struct mcx_dmamem mxm; 5419 struct mcx_cmd_create_flow_table_in *in; 5420 struct mcx_cmd_create_flow_table_mb_in *mbin; 5421 struct mcx_cmd_create_flow_table_out *out; 5422 int error; 5423 int token; 5424 5425 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5426 token = mcx_cmdq_token(sc); 5427 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5428 sizeof(*out), token); 5429 5430 in = mcx_cmdq_in(cqe); 5431 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5432 in->cmd_op_mod = htobe16(0); 5433 5434 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5435 &cqe->cq_input_ptr, token) != 0) { 5436 printf("%s: unable to allocate create flow table mailbox\n", 5437 DEVNAME(sc)); 5438 return (-1); 5439 } 5440 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5441 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5442 mbin->cmd_ctx.ft_log_size = log_size; 5443 mbin->cmd_ctx.ft_level = level; 5444 5445 mcx_cmdq_mboxes_sign(&mxm, 1); 5446 mcx_cmdq_post(sc, cqe, 0); 5447 error = mcx_cmdq_poll(sc, cqe, 1000); 5448 if (error != 0) { 5449 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5450 goto free; 5451 } 5452 if (mcx_cmdq_verify(cqe) != 0) { 5453 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5454 goto free; 5455 } 5456 5457 out = mcx_cmdq_out(cqe); 5458 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5459 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5460 out->cmd_status, betoh32(out->cmd_syndrome)); 5461 error = -1; 5462 goto free; 5463 } 5464 5465 *flow_table_id = mcx_get_id(out->cmd_table_id); 5466 free: 5467 mcx_dmamem_free(sc, &mxm); 5468 return (error); 5469 } 5470 5471 static int 5472 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5473 { 5474 struct mcx_cmdq_entry *cqe; 5475 struct mcx_dmamem mxm; 5476 struct mcx_cmd_set_flow_table_root_in *in; 5477 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5478 struct mcx_cmd_set_flow_table_root_out *out; 5479 int error; 5480 int token; 5481 5482 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5483 token = mcx_cmdq_token(sc); 5484 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5485 sizeof(*out), token); 5486 5487 in = mcx_cmdq_in(cqe); 5488 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5489 in->cmd_op_mod = htobe16(0); 5490 5491 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5492 &cqe->cq_input_ptr, token) != 0) { 5493 printf("%s: unable to allocate set flow table root mailbox\n", 5494 DEVNAME(sc)); 5495 return (-1); 5496 } 5497 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5498 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5499 mbin->cmd_table_id = htobe32(flow_table_id); 5500 5501 mcx_cmdq_mboxes_sign(&mxm, 1); 5502 mcx_cmdq_post(sc, cqe, 0); 5503 error = mcx_cmdq_poll(sc, cqe, 1000); 5504 if (error != 0) { 5505 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5506 goto free; 5507 } 5508 if (mcx_cmdq_verify(cqe) != 0) { 5509 printf("%s: set flow table root command corrupt\n", 5510 DEVNAME(sc)); 5511 goto free; 5512 } 5513 5514 out = mcx_cmdq_out(cqe); 5515 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5516 printf("%s: set flow table root failed (%x, %x)\n", 5517 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5518 error = -1; 5519 goto free; 5520 } 5521 5522 free: 5523 mcx_dmamem_free(sc, &mxm); 5524 return (error); 5525 } 5526 5527 static int 5528 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5529 { 5530 struct mcx_cmdq_entry *cqe; 5531 struct mcx_dmamem mxm; 5532 struct mcx_cmd_destroy_flow_table_in *in; 5533 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5534 struct mcx_cmd_destroy_flow_table_out *out; 5535 int error; 5536 int token; 5537 5538 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5539 token = mcx_cmdq_token(sc); 5540 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5541 5542 in = mcx_cmdq_in(cqe); 5543 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5544 in->cmd_op_mod = htobe16(0); 5545 5546 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5547 &cqe->cq_input_ptr, token) != 0) { 5548 printf("%s: unable to allocate destroy flow table mailbox\n", 5549 DEVNAME(sc)); 5550 return (-1); 5551 } 5552 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5553 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5554 mb->cmd_table_id = htobe32(flow_table_id); 5555 5556 mcx_cmdq_mboxes_sign(&mxm, 1); 5557 mcx_cmdq_post(sc, cqe, 0); 5558 error = mcx_cmdq_poll(sc, cqe, 1000); 5559 if (error != 0) { 5560 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5561 goto free; 5562 } 5563 if (mcx_cmdq_verify(cqe) != 0) { 5564 printf("%s: destroy flow table command corrupt\n", 5565 DEVNAME(sc)); 5566 goto free; 5567 } 5568 5569 out = mcx_cmdq_out(cqe); 5570 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5571 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5572 out->cmd_status, betoh32(out->cmd_syndrome)); 5573 error = -1; 5574 goto free; 5575 } 5576 5577 free: 5578 mcx_dmamem_free(sc, &mxm); 5579 return (error); 5580 } 5581 5582 5583 static int 5584 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5585 int start, int size, int match_enable, struct mcx_flow_match *match) 5586 { 5587 struct mcx_cmdq_entry *cqe; 5588 struct mcx_dmamem mxm; 5589 struct mcx_cmd_create_flow_group_in *in; 5590 struct mcx_cmd_create_flow_group_mb_in *mbin; 5591 struct mcx_cmd_create_flow_group_out *out; 5592 struct mcx_flow_group *mfg; 5593 int error; 5594 int token; 5595 5596 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5597 token = mcx_cmdq_token(sc); 5598 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5599 token); 5600 5601 in = mcx_cmdq_in(cqe); 5602 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5603 in->cmd_op_mod = htobe16(0); 5604 5605 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5606 != 0) { 5607 printf("%s: unable to allocate create flow group mailbox\n", 5608 DEVNAME(sc)); 5609 return (-1); 5610 } 5611 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5612 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5613 mbin->cmd_table_id = htobe32(flow_table_id); 5614 mbin->cmd_start_flow_index = htobe32(start); 5615 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5616 5617 mbin->cmd_match_criteria_enable = match_enable; 5618 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5619 5620 mcx_cmdq_mboxes_sign(&mxm, 2); 5621 mcx_cmdq_post(sc, cqe, 0); 5622 error = mcx_cmdq_poll(sc, cqe, 1000); 5623 if (error != 0) { 5624 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5625 goto free; 5626 } 5627 if (mcx_cmdq_verify(cqe) != 0) { 5628 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5629 goto free; 5630 } 5631 5632 out = mcx_cmdq_out(cqe); 5633 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5634 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5635 out->cmd_status, betoh32(out->cmd_syndrome)); 5636 error = -1; 5637 goto free; 5638 } 5639 5640 mfg = &sc->sc_flow_group[group]; 5641 mfg->g_id = mcx_get_id(out->cmd_group_id); 5642 mfg->g_table = flow_table_id; 5643 mfg->g_start = start; 5644 mfg->g_size = size; 5645 5646 free: 5647 mcx_dmamem_free(sc, &mxm); 5648 return (error); 5649 } 5650 5651 static int 5652 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5653 { 5654 struct mcx_cmdq_entry *cqe; 5655 struct mcx_dmamem mxm; 5656 struct mcx_cmd_destroy_flow_group_in *in; 5657 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5658 struct mcx_cmd_destroy_flow_group_out *out; 5659 struct mcx_flow_group *mfg; 5660 int error; 5661 int token; 5662 5663 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5664 token = mcx_cmdq_token(sc); 5665 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5666 5667 in = mcx_cmdq_in(cqe); 5668 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5669 in->cmd_op_mod = htobe16(0); 5670 5671 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5672 &cqe->cq_input_ptr, token) != 0) { 5673 printf("%s: unable to allocate destroy flow group mailbox\n", 5674 DEVNAME(sc)); 5675 return (-1); 5676 } 5677 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5678 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5679 mfg = &sc->sc_flow_group[group]; 5680 mb->cmd_table_id = htobe32(mfg->g_table); 5681 mb->cmd_group_id = htobe32(mfg->g_id); 5682 5683 mcx_cmdq_mboxes_sign(&mxm, 2); 5684 mcx_cmdq_post(sc, cqe, 0); 5685 error = mcx_cmdq_poll(sc, cqe, 1000); 5686 if (error != 0) { 5687 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5688 goto free; 5689 } 5690 if (mcx_cmdq_verify(cqe) != 0) { 5691 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5692 goto free; 5693 } 5694 5695 out = mcx_cmdq_out(cqe); 5696 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5697 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5698 out->cmd_status, betoh32(out->cmd_syndrome)); 5699 error = -1; 5700 goto free; 5701 } 5702 5703 mfg->g_id = -1; 5704 mfg->g_table = -1; 5705 mfg->g_size = 0; 5706 mfg->g_start = 0; 5707 free: 5708 mcx_dmamem_free(sc, &mxm); 5709 return (error); 5710 } 5711 5712 static int 5713 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5714 uint8_t *macaddr, uint32_t dest) 5715 { 5716 struct mcx_cmdq_entry *cqe; 5717 struct mcx_dmamem mxm; 5718 struct mcx_cmd_set_flow_table_entry_in *in; 5719 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5720 struct mcx_cmd_set_flow_table_entry_out *out; 5721 struct mcx_flow_group *mfg; 5722 uint32_t *pdest; 5723 int error; 5724 int token; 5725 5726 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5727 token = mcx_cmdq_token(sc); 5728 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5729 sizeof(*out), token); 5730 5731 in = mcx_cmdq_in(cqe); 5732 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5733 in->cmd_op_mod = htobe16(0); 5734 5735 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5736 != 0) { 5737 printf("%s: unable to allocate set flow table entry mailbox\n", 5738 DEVNAME(sc)); 5739 return (-1); 5740 } 5741 5742 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5743 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5744 5745 mfg = &sc->sc_flow_group[group]; 5746 mbin->cmd_table_id = htobe32(mfg->g_table); 5747 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5748 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5749 5750 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5751 pdest = (uint32_t *) 5752 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5753 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5754 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5755 *pdest = htobe32(dest); 5756 5757 /* the only thing we match on at the moment is the dest mac address */ 5758 if (macaddr != NULL) { 5759 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5760 ETHER_ADDR_LEN); 5761 } 5762 5763 mcx_cmdq_mboxes_sign(&mxm, 2); 5764 mcx_cmdq_post(sc, cqe, 0); 5765 error = mcx_cmdq_poll(sc, cqe, 1000); 5766 if (error != 0) { 5767 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5768 goto free; 5769 } 5770 if (mcx_cmdq_verify(cqe) != 0) { 5771 printf("%s: set flow table entry command corrupt\n", 5772 DEVNAME(sc)); 5773 goto free; 5774 } 5775 5776 out = mcx_cmdq_out(cqe); 5777 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5778 printf("%s: set flow table entry failed (%x, %x)\n", 5779 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5780 error = -1; 5781 goto free; 5782 } 5783 5784 free: 5785 mcx_dmamem_free(sc, &mxm); 5786 return (error); 5787 } 5788 5789 static int 5790 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5791 int ethertype, int ip_proto, uint32_t dest) 5792 { 5793 struct mcx_cmdq_entry *cqe; 5794 struct mcx_dmamem mxm; 5795 struct mcx_cmd_set_flow_table_entry_in *in; 5796 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5797 struct mcx_cmd_set_flow_table_entry_out *out; 5798 struct mcx_flow_group *mfg; 5799 uint32_t *pdest; 5800 int error; 5801 int token; 5802 5803 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5804 token = mcx_cmdq_token(sc); 5805 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5806 sizeof(*out), token); 5807 5808 in = mcx_cmdq_in(cqe); 5809 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5810 in->cmd_op_mod = htobe16(0); 5811 5812 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5813 != 0) { 5814 printf("%s: unable to allocate set flow table entry mailbox\n", 5815 DEVNAME(sc)); 5816 return (-1); 5817 } 5818 5819 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5820 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5821 5822 mfg = &sc->sc_flow_group[group]; 5823 mbin->cmd_table_id = htobe32(mfg->g_table); 5824 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5825 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5826 5827 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5828 pdest = (uint32_t *) 5829 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5830 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5831 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5832 *pdest = htobe32(dest); 5833 5834 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 5835 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 5836 5837 mcx_cmdq_mboxes_sign(&mxm, 2); 5838 mcx_cmdq_post(sc, cqe, 0); 5839 error = mcx_cmdq_poll(sc, cqe, 1000); 5840 if (error != 0) { 5841 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5842 goto free; 5843 } 5844 if (mcx_cmdq_verify(cqe) != 0) { 5845 printf("%s: set flow table entry command corrupt\n", 5846 DEVNAME(sc)); 5847 goto free; 5848 } 5849 5850 out = mcx_cmdq_out(cqe); 5851 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5852 printf("%s: set flow table entry failed (%x, %x)\n", 5853 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5854 error = -1; 5855 goto free; 5856 } 5857 5858 free: 5859 mcx_dmamem_free(sc, &mxm); 5860 return (error); 5861 } 5862 5863 static int 5864 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 5865 { 5866 struct mcx_cmdq_entry *cqe; 5867 struct mcx_dmamem mxm; 5868 struct mcx_cmd_delete_flow_table_entry_in *in; 5869 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 5870 struct mcx_cmd_delete_flow_table_entry_out *out; 5871 struct mcx_flow_group *mfg; 5872 int error; 5873 int token; 5874 5875 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5876 token = mcx_cmdq_token(sc); 5877 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5878 token); 5879 5880 in = mcx_cmdq_in(cqe); 5881 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 5882 in->cmd_op_mod = htobe16(0); 5883 5884 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5885 &cqe->cq_input_ptr, token) != 0) { 5886 printf("%s: unable to allocate " 5887 "delete flow table entry mailbox\n", DEVNAME(sc)); 5888 return (-1); 5889 } 5890 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5891 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5892 5893 mfg = &sc->sc_flow_group[group]; 5894 mbin->cmd_table_id = htobe32(mfg->g_table); 5895 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5896 5897 mcx_cmdq_mboxes_sign(&mxm, 2); 5898 mcx_cmdq_post(sc, cqe, 0); 5899 error = mcx_cmdq_poll(sc, cqe, 1000); 5900 if (error != 0) { 5901 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 5902 goto free; 5903 } 5904 if (mcx_cmdq_verify(cqe) != 0) { 5905 printf("%s: delete flow table entry command corrupt\n", 5906 DEVNAME(sc)); 5907 goto free; 5908 } 5909 5910 out = mcx_cmdq_out(cqe); 5911 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5912 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 5913 DEVNAME(sc), group, index, out->cmd_status, 5914 betoh32(out->cmd_syndrome)); 5915 error = -1; 5916 goto free; 5917 } 5918 5919 free: 5920 mcx_dmamem_free(sc, &mxm); 5921 return (error); 5922 } 5923 5924 #if 0 5925 int 5926 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 5927 { 5928 struct mcx_dmamem mxm; 5929 struct mcx_cmdq_entry *cqe; 5930 struct mcx_cmd_query_flow_table_in *in; 5931 struct mcx_cmd_query_flow_table_mb_in *mbin; 5932 struct mcx_cmd_query_flow_table_out *out; 5933 struct mcx_cmd_query_flow_table_mb_out *mbout; 5934 uint8_t token = mcx_cmdq_token(sc); 5935 int error; 5936 int i; 5937 uint8_t *dump; 5938 5939 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5940 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5941 sizeof(*out) + sizeof(*mbout) + 16, token); 5942 5943 in = mcx_cmdq_in(cqe); 5944 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 5945 in->cmd_op_mod = htobe16(0); 5946 5947 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5948 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 5949 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5950 &cqe->cq_output_ptr, token) != 0) { 5951 printf(", unable to allocate query flow table mailboxes\n"); 5952 return (-1); 5953 } 5954 cqe->cq_input_ptr = cqe->cq_output_ptr; 5955 5956 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5957 mbin->cmd_table_type = 0; 5958 mbin->cmd_table_id = htobe32(flow_table_id); 5959 5960 mcx_cmdq_mboxes_sign(&mxm, 1); 5961 5962 mcx_cmdq_post(sc, cqe, 0); 5963 error = mcx_cmdq_poll(sc, cqe, 1000); 5964 if (error != 0) { 5965 printf("%s: query flow table timeout\n", DEVNAME(sc)); 5966 goto free; 5967 } 5968 error = mcx_cmdq_verify(cqe); 5969 if (error != 0) { 5970 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 5971 goto free; 5972 } 5973 5974 out = mcx_cmdq_out(cqe); 5975 switch (out->cmd_status) { 5976 case MCX_CQ_STATUS_OK: 5977 break; 5978 default: 5979 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 5980 out->cmd_status, betoh32(out->cmd_syndrome)); 5981 error = -1; 5982 goto free; 5983 } 5984 5985 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 5986 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5987 dump = (uint8_t *)mbout + 8; 5988 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 5989 printf("%.2x ", dump[i]); 5990 if (i % 16 == 15) 5991 printf("\n"); 5992 } 5993 free: 5994 mcx_cq_mboxes_free(sc, &mxm); 5995 return (error); 5996 } 5997 int 5998 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 5999 { 6000 struct mcx_dmamem mxm; 6001 struct mcx_cmdq_entry *cqe; 6002 struct mcx_cmd_query_flow_table_entry_in *in; 6003 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 6004 struct mcx_cmd_query_flow_table_entry_out *out; 6005 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 6006 uint8_t token = mcx_cmdq_token(sc); 6007 int error; 6008 int i; 6009 uint8_t *dump; 6010 6011 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6012 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6013 sizeof(*out) + sizeof(*mbout) + 16, token); 6014 6015 in = mcx_cmdq_in(cqe); 6016 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 6017 in->cmd_op_mod = htobe16(0); 6018 6019 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6020 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6021 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6022 &cqe->cq_output_ptr, token) != 0) { 6023 printf(", unable to allocate " 6024 "query flow table entry mailboxes\n"); 6025 return (-1); 6026 } 6027 cqe->cq_input_ptr = cqe->cq_output_ptr; 6028 6029 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6030 mbin->cmd_table_type = 0; 6031 mbin->cmd_table_id = htobe32(flow_table_id); 6032 mbin->cmd_flow_index = htobe32(index); 6033 6034 mcx_cmdq_mboxes_sign(&mxm, 1); 6035 6036 mcx_cmdq_post(sc, cqe, 0); 6037 error = mcx_cmdq_poll(sc, cqe, 1000); 6038 if (error != 0) { 6039 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 6040 goto free; 6041 } 6042 error = mcx_cmdq_verify(cqe); 6043 if (error != 0) { 6044 printf("%s: query flow table entry reply corrupt\n", 6045 DEVNAME(sc)); 6046 goto free; 6047 } 6048 6049 out = mcx_cmdq_out(cqe); 6050 switch (out->cmd_status) { 6051 case MCX_CQ_STATUS_OK: 6052 break; 6053 default: 6054 printf("%s: query flow table entry failed (%x/%x)\n", 6055 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6056 error = -1; 6057 goto free; 6058 } 6059 6060 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 6061 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6062 dump = (uint8_t *)mbout; 6063 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6064 printf("%.2x ", dump[i]); 6065 if (i % 16 == 15) 6066 printf("\n"); 6067 } 6068 6069 free: 6070 mcx_cq_mboxes_free(sc, &mxm); 6071 return (error); 6072 } 6073 6074 int 6075 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 6076 { 6077 struct mcx_dmamem mxm; 6078 struct mcx_cmdq_entry *cqe; 6079 struct mcx_cmd_query_flow_group_in *in; 6080 struct mcx_cmd_query_flow_group_mb_in *mbin; 6081 struct mcx_cmd_query_flow_group_out *out; 6082 struct mcx_cmd_query_flow_group_mb_out *mbout; 6083 uint8_t token = mcx_cmdq_token(sc); 6084 int error; 6085 int i; 6086 uint8_t *dump; 6087 6088 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6089 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6090 sizeof(*out) + sizeof(*mbout) + 16, token); 6091 6092 in = mcx_cmdq_in(cqe); 6093 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 6094 in->cmd_op_mod = htobe16(0); 6095 6096 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6097 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6098 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6099 &cqe->cq_output_ptr, token) != 0) { 6100 printf(", unable to allocate query flow group mailboxes\n"); 6101 return (-1); 6102 } 6103 cqe->cq_input_ptr = cqe->cq_output_ptr; 6104 6105 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6106 mbin->cmd_table_type = 0; 6107 mbin->cmd_table_id = htobe32(flow_table_id); 6108 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 6109 6110 mcx_cmdq_mboxes_sign(&mxm, 1); 6111 6112 mcx_cmdq_post(sc, cqe, 0); 6113 error = mcx_cmdq_poll(sc, cqe, 1000); 6114 if (error != 0) { 6115 printf("%s: query flow group timeout\n", DEVNAME(sc)); 6116 goto free; 6117 } 6118 error = mcx_cmdq_verify(cqe); 6119 if (error != 0) { 6120 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 6121 goto free; 6122 } 6123 6124 out = mcx_cmdq_out(cqe); 6125 switch (out->cmd_status) { 6126 case MCX_CQ_STATUS_OK: 6127 break; 6128 default: 6129 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 6130 out->cmd_status, betoh32(out->cmd_syndrome)); 6131 error = -1; 6132 goto free; 6133 } 6134 6135 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 6136 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6137 dump = (uint8_t *)mbout; 6138 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6139 printf("%.2x ", dump[i]); 6140 if (i % 16 == 15) 6141 printf("\n"); 6142 } 6143 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 6144 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6145 printf("%.2x ", dump[i]); 6146 if (i % 16 == 15) 6147 printf("\n"); 6148 } 6149 6150 free: 6151 mcx_cq_mboxes_free(sc, &mxm); 6152 return (error); 6153 } 6154 6155 static int 6156 mcx_dump_counters(struct mcx_softc *sc) 6157 { 6158 struct mcx_dmamem mxm; 6159 struct mcx_cmdq_entry *cqe; 6160 struct mcx_cmd_query_vport_counters_in *in; 6161 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6162 struct mcx_cmd_query_vport_counters_out *out; 6163 struct mcx_nic_vport_counters *counters; 6164 int error, token; 6165 6166 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6167 token = mcx_cmdq_token(sc); 6168 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6169 sizeof(*out) + sizeof(*counters), token); 6170 6171 in = mcx_cmdq_in(cqe); 6172 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6173 in->cmd_op_mod = htobe16(0); 6174 6175 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6176 &cqe->cq_output_ptr, token) != 0) { 6177 printf(", unable to allocate " 6178 "query nic vport counters mailboxen\n"); 6179 return (-1); 6180 } 6181 cqe->cq_input_ptr = cqe->cq_output_ptr; 6182 6183 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6184 mbin->cmd_clear = 0x80; 6185 6186 mcx_cmdq_mboxes_sign(&mxm, 1); 6187 mcx_cmdq_post(sc, cqe, 0); 6188 6189 error = mcx_cmdq_poll(sc, cqe, 1000); 6190 if (error != 0) { 6191 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6192 goto free; 6193 } 6194 if (mcx_cmdq_verify(cqe) != 0) { 6195 printf("%s: query nic vport counters command corrupt\n", 6196 DEVNAME(sc)); 6197 goto free; 6198 } 6199 6200 out = mcx_cmdq_out(cqe); 6201 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6202 printf("%s: query nic vport counters failed (%x, %x)\n", 6203 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6204 error = -1; 6205 goto free; 6206 } 6207 6208 counters = (struct mcx_nic_vport_counters *) 6209 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6210 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6211 counters->rx_ucast.packets + counters->tx_ucast.packets + 6212 counters->rx_err.packets + counters->tx_err.packets) 6213 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6214 DEVNAME(sc), 6215 betoh64(counters->tx_err.packets), 6216 betoh64(counters->rx_err.packets), 6217 betoh64(counters->tx_ucast.packets), 6218 betoh64(counters->rx_ucast.packets), 6219 betoh64(counters->tx_bcast.packets), 6220 betoh64(counters->rx_bcast.packets)); 6221 free: 6222 mcx_dmamem_free(sc, &mxm); 6223 6224 return (error); 6225 } 6226 6227 static int 6228 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6229 { 6230 struct mcx_dmamem mxm; 6231 struct mcx_cmdq_entry *cqe; 6232 struct mcx_cmd_query_flow_counter_in *in; 6233 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6234 struct mcx_cmd_query_flow_counter_out *out; 6235 struct mcx_counter *counters; 6236 int error, token; 6237 6238 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6239 token = mcx_cmdq_token(sc); 6240 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6241 sizeof(*counters), token); 6242 6243 in = mcx_cmdq_in(cqe); 6244 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6245 in->cmd_op_mod = htobe16(0); 6246 6247 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6248 &cqe->cq_output_ptr, token) != 0) { 6249 printf(", unable to allocate query flow counter mailboxen\n"); 6250 return (-1); 6251 } 6252 cqe->cq_input_ptr = cqe->cq_output_ptr; 6253 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6254 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6255 mbin->cmd_clear = 0x80; 6256 6257 mcx_cmdq_mboxes_sign(&mxm, 1); 6258 mcx_cmdq_post(sc, cqe, 0); 6259 6260 error = mcx_cmdq_poll(sc, cqe, 1000); 6261 if (error != 0) { 6262 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6263 goto free; 6264 } 6265 if (mcx_cmdq_verify(cqe) != 0) { 6266 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6267 goto free; 6268 } 6269 6270 out = mcx_cmdq_out(cqe); 6271 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6272 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6273 out->cmd_status, betoh32(out->cmd_syndrome)); 6274 error = -1; 6275 goto free; 6276 } 6277 6278 counters = (struct mcx_counter *) 6279 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6280 if (counters->packets) 6281 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6282 betoh64(counters->packets)); 6283 free: 6284 mcx_dmamem_free(sc, &mxm); 6285 6286 return (error); 6287 } 6288 6289 #endif 6290 6291 #if NKSTAT > 0 6292 6293 int 6294 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx) 6295 { 6296 struct mcx_dmamem mxm; 6297 struct mcx_cmdq_entry *cqe; 6298 struct mcx_cmd_query_rq_in *in; 6299 struct mcx_cmd_query_rq_out *out; 6300 struct mcx_cmd_query_rq_mb_out *mbout; 6301 uint8_t token = mcx_cmdq_token(sc); 6302 int error; 6303 6304 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6305 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6306 token); 6307 6308 in = mcx_cmdq_in(cqe); 6309 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 6310 in->cmd_op_mod = htobe16(0); 6311 in->cmd_rqn = htobe32(rx->rx_rqn); 6312 6313 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6314 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6315 &cqe->cq_output_ptr, token) != 0) { 6316 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc)); 6317 return (-1); 6318 } 6319 6320 mcx_cmdq_mboxes_sign(&mxm, 1); 6321 6322 mcx_cmdq_post(sc, cqe, 0); 6323 error = mcx_cmdq_poll(sc, cqe, 1000); 6324 if (error != 0) { 6325 printf("%s: query rq timeout\n", DEVNAME(sc)); 6326 goto free; 6327 } 6328 error = mcx_cmdq_verify(cqe); 6329 if (error != 0) { 6330 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 6331 goto free; 6332 } 6333 6334 out = mcx_cmdq_out(cqe); 6335 switch (out->cmd_status) { 6336 case MCX_CQ_STATUS_OK: 6337 break; 6338 default: 6339 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6340 out->cmd_status, betoh32(out->cmd_syndrome)); 6341 error = -1; 6342 goto free; 6343 } 6344 6345 mbout = (struct mcx_cmd_query_rq_mb_out *) 6346 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6347 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx)); 6348 6349 free: 6350 mcx_cq_mboxes_free(sc, &mxm); 6351 return (error); 6352 } 6353 6354 int 6355 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx) 6356 { 6357 struct mcx_dmamem mxm; 6358 struct mcx_cmdq_entry *cqe; 6359 struct mcx_cmd_query_sq_in *in; 6360 struct mcx_cmd_query_sq_out *out; 6361 struct mcx_cmd_query_sq_mb_out *mbout; 6362 uint8_t token = mcx_cmdq_token(sc); 6363 int error; 6364 6365 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6366 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6367 token); 6368 6369 in = mcx_cmdq_in(cqe); 6370 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6371 in->cmd_op_mod = htobe16(0); 6372 in->cmd_sqn = htobe32(tx->tx_sqn); 6373 6374 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6375 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6376 &cqe->cq_output_ptr, token) != 0) { 6377 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc)); 6378 return (-1); 6379 } 6380 6381 mcx_cmdq_mboxes_sign(&mxm, 1); 6382 6383 mcx_cmdq_post(sc, cqe, 0); 6384 error = mcx_cmdq_poll(sc, cqe, 1000); 6385 if (error != 0) { 6386 printf("%s: query sq timeout\n", DEVNAME(sc)); 6387 goto free; 6388 } 6389 error = mcx_cmdq_verify(cqe); 6390 if (error != 0) { 6391 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6392 goto free; 6393 } 6394 6395 out = mcx_cmdq_out(cqe); 6396 switch (out->cmd_status) { 6397 case MCX_CQ_STATUS_OK: 6398 break; 6399 default: 6400 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6401 out->cmd_status, betoh32(out->cmd_syndrome)); 6402 error = -1; 6403 goto free; 6404 } 6405 6406 mbout = (struct mcx_cmd_query_sq_mb_out *) 6407 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6408 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx)); 6409 6410 free: 6411 mcx_cq_mboxes_free(sc, &mxm); 6412 return (error); 6413 } 6414 6415 int 6416 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx) 6417 { 6418 struct mcx_dmamem mxm; 6419 struct mcx_cmdq_entry *cqe; 6420 struct mcx_cmd_query_cq_in *in; 6421 struct mcx_cmd_query_cq_out *out; 6422 struct mcx_cq_ctx *ctx; 6423 uint8_t token = mcx_cmdq_token(sc); 6424 int error; 6425 6426 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6427 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6428 token); 6429 6430 in = mcx_cmdq_in(cqe); 6431 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ); 6432 in->cmd_op_mod = htobe16(0); 6433 in->cmd_cqn = htobe32(cq->cq_n); 6434 6435 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6436 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6437 &cqe->cq_output_ptr, token) != 0) { 6438 printf("%s: unable to allocate query cq mailboxes\n", DEVNAME(sc)); 6439 return (-1); 6440 } 6441 6442 mcx_cmdq_mboxes_sign(&mxm, 1); 6443 6444 mcx_cmdq_post(sc, cqe, 0); 6445 error = mcx_cmdq_poll(sc, cqe, 1000); 6446 if (error != 0) { 6447 printf("%s: query cq timeout\n", DEVNAME(sc)); 6448 goto free; 6449 } 6450 error = mcx_cmdq_verify(cqe); 6451 if (error != 0) { 6452 printf("%s: query cq reply corrupt\n", DEVNAME(sc)); 6453 goto free; 6454 } 6455 6456 out = mcx_cmdq_out(cqe); 6457 switch (out->cmd_status) { 6458 case MCX_CQ_STATUS_OK: 6459 break; 6460 default: 6461 printf("%s: query cq failed (%x/%x)\n", DEVNAME(sc), 6462 out->cmd_status, betoh32(out->cmd_syndrome)); 6463 error = -1; 6464 goto free; 6465 } 6466 6467 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6468 memcpy(cq_ctx, ctx, sizeof(*cq_ctx)); 6469 free: 6470 mcx_cq_mboxes_free(sc, &mxm); 6471 return (error); 6472 } 6473 6474 int 6475 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx) 6476 { 6477 struct mcx_dmamem mxm; 6478 struct mcx_cmdq_entry *cqe; 6479 struct mcx_cmd_query_eq_in *in; 6480 struct mcx_cmd_query_eq_out *out; 6481 struct mcx_eq_ctx *ctx; 6482 uint8_t token = mcx_cmdq_token(sc); 6483 int error; 6484 6485 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6486 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6487 token); 6488 6489 in = mcx_cmdq_in(cqe); 6490 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ); 6491 in->cmd_op_mod = htobe16(0); 6492 in->cmd_eqn = htobe32(eq->eq_n); 6493 6494 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6495 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6496 &cqe->cq_output_ptr, token) != 0) { 6497 printf("%s: unable to allocate query eq mailboxes\n", DEVNAME(sc)); 6498 return (-1); 6499 } 6500 6501 mcx_cmdq_mboxes_sign(&mxm, 1); 6502 6503 mcx_cmdq_post(sc, cqe, 0); 6504 error = mcx_cmdq_poll(sc, cqe, 1000); 6505 if (error != 0) { 6506 printf("%s: query eq timeout\n", DEVNAME(sc)); 6507 goto free; 6508 } 6509 error = mcx_cmdq_verify(cqe); 6510 if (error != 0) { 6511 printf("%s: query eq reply corrupt\n", DEVNAME(sc)); 6512 goto free; 6513 } 6514 6515 out = mcx_cmdq_out(cqe); 6516 switch (out->cmd_status) { 6517 case MCX_CQ_STATUS_OK: 6518 break; 6519 default: 6520 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc), 6521 out->cmd_status, betoh32(out->cmd_syndrome)); 6522 error = -1; 6523 goto free; 6524 } 6525 6526 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6527 memcpy(eq_ctx, ctx, sizeof(*eq_ctx)); 6528 free: 6529 mcx_cq_mboxes_free(sc, &mxm); 6530 return (error); 6531 } 6532 6533 #endif /* NKSTAT > 0 */ 6534 6535 static inline unsigned int 6536 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots) 6537 { 6538 struct mcx_rq_entry *ring, *rqe; 6539 struct mcx_slot *ms; 6540 struct mbuf *m; 6541 uint slot, p, fills; 6542 6543 ring = MCX_DMA_KVA(&rx->rx_rq_mem); 6544 p = rx->rx_prod; 6545 6546 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6547 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 6548 6549 for (fills = 0; fills < nslots; fills++) { 6550 slot = p % (1 << MCX_LOG_RQ_SIZE); 6551 6552 ms = &rx->rx_slots[slot]; 6553 rqe = &ring[slot]; 6554 6555 m = MCLGETL(NULL, M_DONTWAIT, sc->sc_rxbufsz); 6556 if (m == NULL) 6557 break; 6558 6559 m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz); 6560 m->m_data += ETHER_ALIGN; 6561 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6562 6563 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6564 BUS_DMA_NOWAIT) != 0) { 6565 m_freem(m); 6566 break; 6567 } 6568 ms->ms_m = m; 6569 6570 htobem32(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len); 6571 htobem64(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr); 6572 htobem32(&rqe->rqe_lkey, sc->sc_lkey); 6573 6574 p++; 6575 } 6576 6577 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6578 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 6579 6580 rx->rx_prod = p; 6581 6582 htobem32(rx->rx_doorbell, p & MCX_WQ_DOORBELL_MASK); 6583 6584 return (nslots - fills); 6585 } 6586 6587 int 6588 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6589 { 6590 u_int slots; 6591 6592 slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6593 if (slots == 0) 6594 return (1); 6595 6596 slots = mcx_rx_fill_slots(sc, rx, slots); 6597 if_rxr_put(&rx->rx_rxr, slots); 6598 return (0); 6599 } 6600 6601 void 6602 mcx_refill(void *xrx) 6603 { 6604 struct mcx_rx *rx = xrx; 6605 struct mcx_softc *sc = rx->rx_softc; 6606 6607 mcx_rx_fill(sc, rx); 6608 6609 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6610 timeout_add(&rx->rx_refill, 1); 6611 } 6612 6613 static int 6614 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6615 struct mcx_cq_entry *cqe) 6616 { 6617 struct mcx_slot *ms; 6618 bus_dmamap_t map; 6619 int slot, slots; 6620 6621 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6622 6623 ms = &tx->tx_slots[slot]; 6624 map = ms->ms_map; 6625 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6626 BUS_DMASYNC_POSTWRITE); 6627 6628 slots = 1; 6629 if (map->dm_nsegs > 1) 6630 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6631 6632 bus_dmamap_unload(sc->sc_dmat, map); 6633 m_freem(ms->ms_m); 6634 ms->ms_m = NULL; 6635 6636 return (slots); 6637 } 6638 6639 static uint64_t 6640 mcx_uptime(void) 6641 { 6642 struct timespec ts; 6643 6644 nanouptime(&ts); 6645 6646 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6647 } 6648 6649 static void 6650 mcx_calibrate_first(struct mcx_softc *sc) 6651 { 6652 struct mcx_calibration *c = &sc->sc_calibration[0]; 6653 int s; 6654 6655 sc->sc_calibration_gen = 0; 6656 6657 s = splhigh(); /* crit_enter? */ 6658 c->c_ubase = mcx_uptime(); 6659 c->c_tbase = mcx_timer(sc); 6660 splx(s); 6661 c->c_ratio = 0; 6662 6663 #ifdef notyet 6664 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 6665 #endif 6666 } 6667 6668 #define MCX_TIMESTAMP_SHIFT 24 6669 6670 static void 6671 mcx_calibrate(void *arg) 6672 { 6673 struct mcx_softc *sc = arg; 6674 struct mcx_calibration *nc, *pc; 6675 uint64_t udiff, tdiff; 6676 unsigned int gen; 6677 int s; 6678 6679 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 6680 return; 6681 6682 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 6683 6684 gen = sc->sc_calibration_gen; 6685 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6686 gen++; 6687 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6688 6689 nc->c_uptime = pc->c_ubase; 6690 nc->c_timestamp = pc->c_tbase; 6691 6692 s = splhigh(); /* crit_enter? */ 6693 nc->c_ubase = mcx_uptime(); 6694 nc->c_tbase = mcx_timer(sc); 6695 splx(s); 6696 6697 udiff = nc->c_ubase - nc->c_uptime; 6698 tdiff = nc->c_tbase - nc->c_timestamp; 6699 6700 /* 6701 * udiff is the wall clock time between calibration ticks, 6702 * which should be 32 seconds or 32 billion nanoseconds. if 6703 * we squint, 1 billion nanoseconds is kind of like a 32 bit 6704 * number, so 32 billion should still have a lot of high bits 6705 * spare. we use this space by shifting the nanoseconds up 6706 * 24 bits so we have a nice big number to divide by the 6707 * number of mcx timer ticks. 6708 */ 6709 nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff; 6710 6711 membar_producer(); 6712 sc->sc_calibration_gen = gen; 6713 } 6714 6715 static int 6716 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6717 struct mcx_cq_entry *cqe, struct mbuf_list *ml, 6718 const struct mcx_calibration *c) 6719 { 6720 struct mcx_slot *ms; 6721 struct mbuf *m; 6722 uint32_t flags; 6723 int slot; 6724 6725 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6726 6727 ms = &rx->rx_slots[slot]; 6728 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, 6729 BUS_DMASYNC_POSTREAD); 6730 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6731 6732 m = ms->ms_m; 6733 ms->ms_m = NULL; 6734 6735 m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); 6736 6737 if (cqe->cq_rx_hash_type) { 6738 m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); 6739 m->m_pkthdr.csum_flags |= M_FLOWID; 6740 } 6741 6742 flags = bemtoh32(&cqe->cq_flags); 6743 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) 6744 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 6745 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) 6746 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK | 6747 M_UDP_CSUM_IN_OK; 6748 #if NVLAN > 0 6749 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6750 m->m_pkthdr.ether_vtag = (flags & 6751 MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6752 m->m_flags |= M_VLANTAG; 6753 } 6754 #endif 6755 6756 #ifdef notyet 6757 if (ISSET(sc->sc_ac.ac_if.if_flags, IFF_LINK0) && c->c_ratio) { 6758 uint64_t t = bemtoh64(&cqe->cq_timestamp); 6759 t -= c->c_timestamp; 6760 t *= c->c_ratio; 6761 t >>= MCX_TIMESTAMP_SHIFT; 6762 t += c->c_uptime; 6763 6764 m->m_pkthdr.ph_timestamp = t; 6765 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6766 } 6767 #endif 6768 6769 ml_enqueue(ml, m); 6770 6771 return (1); 6772 } 6773 6774 static struct mcx_cq_entry * 6775 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6776 { 6777 struct mcx_cq_entry *cqe; 6778 int next; 6779 6780 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 6781 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 6782 6783 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 6784 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 6785 return (&cqe[next]); 6786 } 6787 6788 return (NULL); 6789 } 6790 6791 static void 6792 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 6793 { 6794 bus_size_t offset; 6795 uint32_t val; 6796 uint64_t uval; 6797 6798 offset = (MCX_PAGE_SIZE * uar); 6799 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 6800 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6801 6802 cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6803 cq->cq_doorbell[1] = htobe32(val); 6804 6805 uval = val; 6806 uval <<= 32; 6807 uval |= cq->cq_n; 6808 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 6809 offset + MCX_UAR_CQ_DOORBELL, htobe64(uval)); 6810 mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t), 6811 BUS_SPACE_BARRIER_WRITE); 6812 } 6813 6814 void 6815 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 6816 { 6817 struct mcx_rx *rx = &q->q_rx; 6818 struct mcx_tx *tx = &q->q_tx; 6819 const struct mcx_calibration *c; 6820 unsigned int gen; 6821 struct mcx_cq_entry *cqe; 6822 uint8_t *cqp; 6823 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 6824 int rxfree, txfree; 6825 6826 gen = sc->sc_calibration_gen; 6827 membar_consumer(); 6828 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6829 6830 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 6831 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 6832 6833 rxfree = 0; 6834 txfree = 0; 6835 while ((cqe = mcx_next_cq_entry(sc, cq))) { 6836 uint8_t opcode; 6837 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 6838 switch (opcode) { 6839 case MCX_CQ_ENTRY_OPCODE_REQ: 6840 txfree += mcx_process_txeof(sc, tx, cqe); 6841 break; 6842 case MCX_CQ_ENTRY_OPCODE_SEND: 6843 rxfree += mcx_process_rx(sc, rx, cqe, &ml, c); 6844 break; 6845 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 6846 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 6847 cqp = (uint8_t *)cqe; 6848 /* printf("%s: cq completion error: %x\n", 6849 DEVNAME(sc), cqp[0x37]); */ 6850 break; 6851 6852 default: 6853 /* printf("%s: cq completion opcode %x??\n", 6854 DEVNAME(sc), opcode); */ 6855 break; 6856 } 6857 6858 cq->cq_cons++; 6859 } 6860 6861 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 6862 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 6863 6864 cq->cq_count++; 6865 mcx_arm_cq(sc, cq, q->q_uar); 6866 6867 if (rxfree > 0) { 6868 if_rxr_put(&rx->rx_rxr, rxfree); 6869 if (ifiq_input(rx->rx_ifiq, &ml)) 6870 if_rxr_livelocked(&rx->rx_rxr); 6871 6872 mcx_rx_fill(sc, rx); 6873 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6874 timeout_add(&rx->rx_refill, 1); 6875 } 6876 if (txfree > 0) { 6877 tx->tx_cons += txfree; 6878 if (ifq_is_oactive(tx->tx_ifq)) 6879 ifq_restart(tx->tx_ifq); 6880 } 6881 } 6882 6883 6884 static void 6885 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 6886 { 6887 bus_size_t offset; 6888 uint32_t val; 6889 6890 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 6891 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 6892 6893 mcx_wr(sc, offset, val); 6894 mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE); 6895 } 6896 6897 static struct mcx_eq_entry * 6898 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 6899 { 6900 struct mcx_eq_entry *eqe; 6901 int next; 6902 6903 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 6904 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 6905 if ((eqe[next].eq_owner & 1) == 6906 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 6907 eq->eq_cons++; 6908 return (&eqe[next]); 6909 } 6910 return (NULL); 6911 } 6912 6913 int 6914 mcx_admin_intr(void *xsc) 6915 { 6916 struct mcx_softc *sc = (struct mcx_softc *)xsc; 6917 struct mcx_eq *eq = &sc->sc_admin_eq; 6918 struct mcx_eq_entry *eqe; 6919 6920 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6921 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 6922 6923 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 6924 switch (eqe->eq_event_type) { 6925 case MCX_EVENT_TYPE_LAST_WQE: 6926 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 6927 break; 6928 6929 case MCX_EVENT_TYPE_CQ_ERROR: 6930 /* printf("%s: cq error\n", DEVNAME(sc)); */ 6931 break; 6932 6933 case MCX_EVENT_TYPE_CMD_COMPLETION: 6934 /* wakeup probably */ 6935 break; 6936 6937 case MCX_EVENT_TYPE_PORT_CHANGE: 6938 task_add(systq, &sc->sc_port_change); 6939 break; 6940 6941 default: 6942 /* printf("%s: something happened\n", DEVNAME(sc)); */ 6943 break; 6944 } 6945 } 6946 6947 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6948 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 6949 6950 mcx_arm_eq(sc, eq, sc->sc_uar); 6951 6952 return (1); 6953 } 6954 6955 int 6956 mcx_cq_intr(void *xq) 6957 { 6958 struct mcx_queues *q = (struct mcx_queues *)xq; 6959 struct mcx_softc *sc = q->q_sc; 6960 struct mcx_eq *eq = &q->q_eq; 6961 struct mcx_eq_entry *eqe; 6962 int cqn; 6963 6964 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6965 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 6966 6967 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 6968 switch (eqe->eq_event_type) { 6969 case MCX_EVENT_TYPE_COMPLETION: 6970 cqn = betoh32(eqe->eq_event_data[6]); 6971 if (cqn == q->q_cq.cq_n) 6972 mcx_process_cq(sc, q, &q->q_cq); 6973 break; 6974 } 6975 } 6976 6977 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 6978 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 6979 6980 mcx_arm_eq(sc, eq, q->q_uar); 6981 6982 return (1); 6983 } 6984 6985 static void 6986 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 6987 int total) 6988 { 6989 struct mcx_slot *ms; 6990 6991 int i = allocated; 6992 while (i-- > 0) { 6993 ms = &slots[i]; 6994 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 6995 if (ms->ms_m != NULL) 6996 m_freem(ms->ms_m); 6997 } 6998 free(slots, M_DEVBUF, total * sizeof(*ms)); 6999 } 7000 7001 static int 7002 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 7003 { 7004 struct mcx_rx *rx; 7005 struct mcx_tx *tx; 7006 struct mcx_slot *ms; 7007 int i; 7008 7009 rx = &q->q_rx; 7010 rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 7011 M_DEVBUF, M_WAITOK | M_ZERO); 7012 if (rx->rx_slots == NULL) { 7013 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 7014 return ENOMEM; 7015 } 7016 7017 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 7018 ms = &rx->rx_slots[i]; 7019 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 7020 sc->sc_hardmtu, 0, 7021 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7022 &ms->ms_map) != 0) { 7023 printf("%s: failed to allocate rx dma maps\n", 7024 DEVNAME(sc)); 7025 goto destroy_rx_slots; 7026 } 7027 } 7028 7029 tx = &q->q_tx; 7030 tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 7031 M_DEVBUF, M_WAITOK | M_ZERO); 7032 if (tx->tx_slots == NULL) { 7033 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 7034 goto destroy_rx_slots; 7035 } 7036 7037 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 7038 ms = &tx->tx_slots[i]; 7039 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 7040 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 7041 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7042 &ms->ms_map) != 0) { 7043 printf("%s: failed to allocate tx dma maps\n", 7044 DEVNAME(sc)); 7045 goto destroy_tx_slots; 7046 } 7047 } 7048 7049 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 7050 q->q_eq.eq_n) != 0) 7051 goto destroy_tx_slots; 7052 7053 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 7054 != 0) 7055 goto destroy_cq; 7056 7057 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 7058 goto destroy_sq; 7059 7060 return 0; 7061 7062 destroy_sq: 7063 mcx_destroy_sq(sc, tx); 7064 destroy_cq: 7065 mcx_destroy_cq(sc, &q->q_cq); 7066 destroy_tx_slots: 7067 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 7068 tx->tx_slots = NULL; 7069 7070 i = (1 << MCX_LOG_RQ_SIZE); 7071 destroy_rx_slots: 7072 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 7073 rx->rx_slots = NULL; 7074 return ENOMEM; 7075 } 7076 7077 static int 7078 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 7079 { 7080 int i; 7081 int count; 7082 7083 count = 0; 7084 for (i = 0; i < nitems(mcx_rss_config); i++) { 7085 if (mcx_rss_config[i].flow_group == group) 7086 count++; 7087 } 7088 7089 return count; 7090 } 7091 7092 static int 7093 mcx_up(struct mcx_softc *sc) 7094 { 7095 struct ifnet *ifp = &sc->sc_ac.ac_if; 7096 struct mcx_rx *rx; 7097 struct mcx_tx *tx; 7098 int i, start, count, flow_group, flow_index; 7099 struct mcx_flow_match match_crit; 7100 struct mcx_rss_rule *rss; 7101 uint32_t dest; 7102 int rqns[MCX_MAX_QUEUES]; 7103 7104 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 7105 goto down; 7106 7107 for (i = 0; i < sc->sc_nqueues; i++) { 7108 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 7109 goto down; 7110 } 7111 } 7112 7113 /* RSS flow table and flow groups */ 7114 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 7115 &sc->sc_rss_flow_table_id) != 0) 7116 goto down; 7117 7118 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7119 sc->sc_rss_flow_table_id; 7120 7121 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 7122 memset(&match_crit, 0, sizeof(match_crit)); 7123 match_crit.mc_ethertype = 0xffff; 7124 match_crit.mc_ip_proto = 0xff; 7125 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 7126 start = 0; 7127 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 7128 if (count != 0) { 7129 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7130 MCX_FLOW_GROUP_RSS_L4, start, count, 7131 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7132 goto down; 7133 start += count; 7134 } 7135 7136 /* L3 RSS flow group (v4/v6, including fragments) */ 7137 memset(&match_crit, 0, sizeof(match_crit)); 7138 match_crit.mc_ethertype = 0xffff; 7139 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 7140 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7141 MCX_FLOW_GROUP_RSS_L3, start, count, 7142 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7143 goto down; 7144 start += count; 7145 7146 /* non-RSS flow group */ 7147 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 7148 memset(&match_crit, 0, sizeof(match_crit)); 7149 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7150 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 7151 goto down; 7152 7153 /* Root flow table, matching packets based on mac address */ 7154 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 7155 &sc->sc_mac_flow_table_id) != 0) 7156 goto down; 7157 7158 /* promisc flow group */ 7159 start = 0; 7160 memset(&match_crit, 0, sizeof(match_crit)); 7161 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7162 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 7163 goto down; 7164 sc->sc_promisc_flow_enabled = 0; 7165 start++; 7166 7167 /* all multicast flow group */ 7168 match_crit.mc_dest_mac[0] = 0x01; 7169 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7170 MCX_FLOW_GROUP_ALLMULTI, start, 1, 7171 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7172 goto down; 7173 sc->sc_allmulti_flow_enabled = 0; 7174 start++; 7175 7176 /* mac address matching flow group */ 7177 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 7178 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7179 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 7180 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7181 goto down; 7182 7183 /* flow table entries for unicast and broadcast */ 7184 start = 0; 7185 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7186 sc->sc_ac.ac_enaddr, dest) != 0) 7187 goto down; 7188 start++; 7189 7190 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7191 etherbroadcastaddr, dest) != 0) 7192 goto down; 7193 start++; 7194 7195 /* multicast entries go after that */ 7196 sc->sc_mcast_flow_base = start; 7197 7198 /* re-add any existing multicast flows */ 7199 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7200 if (sc->sc_mcast_flows[i][0] != 0) { 7201 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 7202 sc->sc_mcast_flow_base + i, 7203 sc->sc_mcast_flows[i], dest); 7204 } 7205 } 7206 7207 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 7208 goto down; 7209 7210 /* 7211 * the RQT can be any size as long as it's a power of two. 7212 * since we also restrict the number of queues to a power of two, 7213 * we can just put each rx queue in once. 7214 */ 7215 for (i = 0; i < sc->sc_nqueues; i++) 7216 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 7217 7218 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) 7219 goto down; 7220 7221 start = 0; 7222 flow_index = 0; 7223 flow_group = -1; 7224 for (i = 0; i < nitems(mcx_rss_config); i++) { 7225 rss = &mcx_rss_config[i]; 7226 if (rss->flow_group != flow_group) { 7227 flow_group = rss->flow_group; 7228 flow_index = 0; 7229 } 7230 7231 if (rss->hash_sel == 0) { 7232 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 7233 &sc->sc_tir[i]) != 0) 7234 goto down; 7235 } else { 7236 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 7237 rss->hash_sel, &sc->sc_tir[i]) != 0) 7238 goto down; 7239 } 7240 7241 if (mcx_set_flow_table_entry_proto(sc, flow_group, 7242 flow_index, rss->ethertype, rss->ip_proto, 7243 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 7244 goto down; 7245 flow_index++; 7246 } 7247 7248 for (i = 0; i < sc->sc_nqueues; i++) { 7249 struct mcx_queues *q = &sc->sc_queues[i]; 7250 rx = &q->q_rx; 7251 tx = &q->q_tx; 7252 7253 /* start the queues */ 7254 if (mcx_ready_sq(sc, tx) != 0) 7255 goto down; 7256 7257 if (mcx_ready_rq(sc, rx) != 0) 7258 goto down; 7259 7260 if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 7261 rx->rx_prod = 0; 7262 mcx_rx_fill(sc, rx); 7263 7264 tx->tx_cons = 0; 7265 tx->tx_prod = 0; 7266 ifq_clr_oactive(tx->tx_ifq); 7267 } 7268 7269 mcx_calibrate_first(sc); 7270 7271 SET(ifp->if_flags, IFF_RUNNING); 7272 7273 return ENETRESET; 7274 down: 7275 mcx_down(sc); 7276 return ENOMEM; 7277 } 7278 7279 static void 7280 mcx_down(struct mcx_softc *sc) 7281 { 7282 struct ifnet *ifp = &sc->sc_ac.ac_if; 7283 struct mcx_rss_rule *rss; 7284 int group, i, flow_group, flow_index; 7285 7286 CLR(ifp->if_flags, IFF_RUNNING); 7287 7288 /* 7289 * delete flow table entries first, so no packets can arrive 7290 * after the barriers 7291 */ 7292 if (sc->sc_promisc_flow_enabled) 7293 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 7294 if (sc->sc_allmulti_flow_enabled) 7295 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 7296 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 7297 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 7298 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7299 if (sc->sc_mcast_flows[i][0] != 0) { 7300 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 7301 sc->sc_mcast_flow_base + i); 7302 } 7303 } 7304 7305 flow_group = -1; 7306 flow_index = 0; 7307 for (i = 0; i < nitems(mcx_rss_config); i++) { 7308 rss = &mcx_rss_config[i]; 7309 if (rss->flow_group != flow_group) { 7310 flow_group = rss->flow_group; 7311 flow_index = 0; 7312 } 7313 7314 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 7315 7316 mcx_destroy_tir(sc, sc->sc_tir[i]); 7317 sc->sc_tir[i] = 0; 7318 7319 flow_index++; 7320 } 7321 intr_barrier(sc->sc_ihc); 7322 for (i = 0; i < sc->sc_nqueues; i++) { 7323 struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; 7324 ifq_barrier(ifq); 7325 7326 timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill); 7327 7328 intr_barrier(sc->sc_queues[i].q_ihc); 7329 } 7330 7331 timeout_del_barrier(&sc->sc_calibrate); 7332 7333 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 7334 if (sc->sc_flow_group[group].g_id != -1) 7335 mcx_destroy_flow_group(sc, group); 7336 } 7337 7338 if (sc->sc_mac_flow_table_id != -1) { 7339 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 7340 sc->sc_mac_flow_table_id = -1; 7341 } 7342 if (sc->sc_rss_flow_table_id != -1) { 7343 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 7344 sc->sc_rss_flow_table_id = -1; 7345 } 7346 if (sc->sc_rqt != -1) { 7347 mcx_destroy_rqt(sc, sc->sc_rqt); 7348 sc->sc_rqt = -1; 7349 } 7350 7351 for (i = 0; i < sc->sc_nqueues; i++) { 7352 struct mcx_queues *q = &sc->sc_queues[i]; 7353 struct mcx_rx *rx = &q->q_rx; 7354 struct mcx_tx *tx = &q->q_tx; 7355 struct mcx_cq *cq = &q->q_cq; 7356 7357 if (rx->rx_rqn != 0) 7358 mcx_destroy_rq(sc, rx); 7359 7360 if (tx->tx_sqn != 0) 7361 mcx_destroy_sq(sc, tx); 7362 7363 if (tx->tx_slots != NULL) { 7364 mcx_free_slots(sc, tx->tx_slots, 7365 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7366 tx->tx_slots = NULL; 7367 } 7368 if (rx->rx_slots != NULL) { 7369 mcx_free_slots(sc, rx->rx_slots, 7370 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7371 rx->rx_slots = NULL; 7372 } 7373 7374 if (cq->cq_n != 0) 7375 mcx_destroy_cq(sc, cq); 7376 } 7377 if (sc->sc_tis != 0) { 7378 mcx_destroy_tis(sc, sc->sc_tis); 7379 sc->sc_tis = 0; 7380 } 7381 } 7382 7383 static int 7384 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 7385 { 7386 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7387 struct ifreq *ifr = (struct ifreq *)data; 7388 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7389 int s, i, error = 0; 7390 uint32_t dest; 7391 7392 s = splnet(); 7393 switch (cmd) { 7394 case SIOCSIFADDR: 7395 ifp->if_flags |= IFF_UP; 7396 /* FALLTHROUGH */ 7397 7398 case SIOCSIFFLAGS: 7399 if (ISSET(ifp->if_flags, IFF_UP)) { 7400 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7401 error = ENETRESET; 7402 else 7403 error = mcx_up(sc); 7404 } else { 7405 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7406 mcx_down(sc); 7407 } 7408 break; 7409 7410 case SIOCGIFMEDIA: 7411 case SIOCSIFMEDIA: 7412 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 7413 break; 7414 7415 case SIOCGIFSFFPAGE: 7416 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 7417 break; 7418 7419 case SIOCGIFRXR: 7420 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 7421 break; 7422 7423 case SIOCADDMULTI: 7424 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 7425 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7426 if (error != 0) 7427 return (error); 7428 7429 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7430 sc->sc_rss_flow_table_id; 7431 7432 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7433 if (sc->sc_mcast_flows[i][0] == 0) { 7434 memcpy(sc->sc_mcast_flows[i], addrlo, 7435 ETHER_ADDR_LEN); 7436 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7437 mcx_set_flow_table_entry_mac(sc, 7438 MCX_FLOW_GROUP_MAC, 7439 sc->sc_mcast_flow_base + i, 7440 sc->sc_mcast_flows[i], dest); 7441 } 7442 break; 7443 } 7444 } 7445 7446 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7447 if (i == MCX_NUM_MCAST_FLOWS) { 7448 SET(ifp->if_flags, IFF_ALLMULTI); 7449 sc->sc_extra_mcast++; 7450 error = ENETRESET; 7451 } 7452 7453 if (sc->sc_ac.ac_multirangecnt > 0) { 7454 SET(ifp->if_flags, IFF_ALLMULTI); 7455 error = ENETRESET; 7456 } 7457 } 7458 } 7459 break; 7460 7461 case SIOCDELMULTI: 7462 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 7463 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7464 if (error != 0) 7465 return (error); 7466 7467 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7468 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7469 ETHER_ADDR_LEN) == 0) { 7470 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7471 mcx_delete_flow_table_entry(sc, 7472 MCX_FLOW_GROUP_MAC, 7473 sc->sc_mcast_flow_base + i); 7474 } 7475 sc->sc_mcast_flows[i][0] = 0; 7476 break; 7477 } 7478 } 7479 7480 if (i == MCX_NUM_MCAST_FLOWS) 7481 sc->sc_extra_mcast--; 7482 7483 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7484 (sc->sc_extra_mcast == 0) && 7485 (sc->sc_ac.ac_multirangecnt == 0)) { 7486 CLR(ifp->if_flags, IFF_ALLMULTI); 7487 error = ENETRESET; 7488 } 7489 } 7490 break; 7491 7492 default: 7493 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 7494 } 7495 7496 if (error == ENETRESET) { 7497 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7498 (IFF_UP | IFF_RUNNING)) 7499 mcx_iff(sc); 7500 error = 0; 7501 } 7502 splx(s); 7503 7504 return (error); 7505 } 7506 7507 static int 7508 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7509 { 7510 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7511 struct mcx_reg_mcia mcia; 7512 struct mcx_reg_pmlp pmlp; 7513 int offset, error; 7514 7515 /* get module number */ 7516 memset(&pmlp, 0, sizeof(pmlp)); 7517 pmlp.rp_local_port = 1; 7518 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7519 sizeof(pmlp)); 7520 if (error != 0) { 7521 printf("%s: unable to get eeprom module number\n", 7522 DEVNAME(sc)); 7523 return error; 7524 } 7525 7526 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7527 memset(&mcia, 0, sizeof(mcia)); 7528 mcia.rm_l = 0; 7529 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 7530 MCX_PMLP_MODULE_NUM_MASK; 7531 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7532 mcia.rm_page_num = sff->sff_page; 7533 mcia.rm_dev_addr = htobe16(offset); 7534 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7535 7536 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7537 &mcia, sizeof(mcia)); 7538 if (error != 0) { 7539 printf("%s: unable to read eeprom at %x\n", 7540 DEVNAME(sc), offset); 7541 return error; 7542 } 7543 7544 memcpy(sff->sff_data + offset, mcia.rm_data, 7545 MCX_MCIA_EEPROM_BYTES); 7546 } 7547 7548 return 0; 7549 } 7550 7551 static int 7552 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 7553 { 7554 struct if_rxring_info *ifrs; 7555 unsigned int i; 7556 int error; 7557 7558 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP, 7559 M_WAITOK|M_ZERO|M_CANFAIL); 7560 if (ifrs == NULL) 7561 return (ENOMEM); 7562 7563 for (i = 0; i < sc->sc_nqueues; i++) { 7564 struct mcx_rx *rx = &sc->sc_queues[i].q_rx; 7565 struct if_rxring_info *ifr = &ifrs[i]; 7566 7567 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 7568 ifr->ifr_size = sc->sc_hardmtu; 7569 ifr->ifr_info = rx->rx_rxr; 7570 } 7571 7572 error = if_rxr_info_ioctl(ifri, i, ifrs); 7573 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 7574 7575 return (error); 7576 } 7577 7578 int 7579 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7580 { 7581 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7582 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7583 case 0: 7584 break; 7585 7586 case EFBIG: 7587 if (m_defrag(m, M_DONTWAIT) == 0 && 7588 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7589 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7590 break; 7591 7592 default: 7593 return (1); 7594 } 7595 7596 ms->ms_m = m; 7597 return (0); 7598 } 7599 7600 static void 7601 mcx_start(struct ifqueue *ifq) 7602 { 7603 struct mcx_tx *tx = ifq->ifq_softc; 7604 struct ifnet *ifp = ifq->ifq_if; 7605 struct mcx_softc *sc = ifp->if_softc; 7606 struct mcx_sq_entry *sq, *sqe; 7607 struct mcx_sq_entry_seg *sqs; 7608 struct mcx_slot *ms; 7609 bus_dmamap_t map; 7610 struct mbuf *m; 7611 u_int idx, free, used; 7612 uint64_t *bf; 7613 uint32_t csum; 7614 size_t bf_base; 7615 int i, seg, nseg; 7616 7617 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7618 7619 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7620 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7621 7622 used = 0; 7623 bf = NULL; 7624 7625 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7626 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 7627 7628 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7629 7630 for (;;) { 7631 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7632 ifq_set_oactive(ifq); 7633 break; 7634 } 7635 7636 m = ifq_dequeue(ifq); 7637 if (m == NULL) { 7638 break; 7639 } 7640 7641 sqe = sq + idx; 7642 ms = &tx->tx_slots[idx]; 7643 memset(sqe, 0, sizeof(*sqe)); 7644 7645 /* ctrl segment */ 7646 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7647 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7648 /* always generate a completion event */ 7649 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7650 7651 /* eth segment */ 7652 csum = 0; 7653 if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 7654 csum |= MCX_SQE_L3_CSUM; 7655 if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) 7656 csum |= MCX_SQE_L4_CSUM; 7657 sqe->sqe_mss_csum = htobe32(csum); 7658 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7659 #if NVLAN > 0 7660 if (m->m_flags & M_VLANTAG) { 7661 struct ether_vlan_header *evh; 7662 evh = (struct ether_vlan_header *) 7663 &sqe->sqe_inline_headers; 7664 7665 /* slightly cheaper vlan_inject() */ 7666 m_copydata(m, 0, ETHER_HDR_LEN, (caddr_t)evh); 7667 evh->evl_proto = evh->evl_encap_proto; 7668 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7669 evh->evl_tag = htons(m->m_pkthdr.ether_vtag); 7670 7671 m_adj(m, ETHER_HDR_LEN); 7672 } else 7673 #endif 7674 { 7675 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7676 (caddr_t)sqe->sqe_inline_headers); 7677 m_adj(m, MCX_SQ_INLINE_SIZE); 7678 } 7679 7680 if (mcx_load_mbuf(sc, ms, m) != 0) { 7681 m_freem(m); 7682 ifp->if_oerrors++; 7683 continue; 7684 } 7685 bf = (uint64_t *)sqe; 7686 7687 #if NBPFILTER > 0 7688 if (ifp->if_bpf) 7689 bpf_mtap_hdr(ifp->if_bpf, 7690 (caddr_t)sqe->sqe_inline_headers, 7691 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 7692 #endif 7693 map = ms->ms_map; 7694 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7695 BUS_DMASYNC_PREWRITE); 7696 7697 sqe->sqe_ds_sq_num = 7698 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7699 (map->dm_nsegs + 3)); 7700 7701 /* data segment - first wqe has one segment */ 7702 sqs = sqe->sqe_segs; 7703 seg = 0; 7704 nseg = 1; 7705 for (i = 0; i < map->dm_nsegs; i++) { 7706 if (seg == nseg) { 7707 /* next slot */ 7708 idx++; 7709 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7710 idx = 0; 7711 tx->tx_prod++; 7712 used++; 7713 7714 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7715 seg = 0; 7716 nseg = MCX_SQ_SEGS_PER_SLOT; 7717 } 7718 sqs[seg].sqs_byte_count = 7719 htobe32(map->dm_segs[i].ds_len); 7720 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7721 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7722 seg++; 7723 } 7724 7725 idx++; 7726 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7727 idx = 0; 7728 tx->tx_prod++; 7729 used++; 7730 } 7731 7732 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7733 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 7734 7735 if (used) { 7736 htobem32(tx->tx_doorbell, tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7737 7738 membar_sync(); 7739 7740 /* 7741 * write the first 64 bits of the last sqe we produced 7742 * to the blue flame buffer 7743 */ 7744 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 7745 bf_base + tx->tx_bf_offset, *bf); 7746 /* next write goes to the other buffer */ 7747 tx->tx_bf_offset ^= sc->sc_bf_size; 7748 7749 membar_sync(); 7750 } 7751 } 7752 7753 static void 7754 mcx_watchdog(struct ifnet *ifp) 7755 { 7756 } 7757 7758 static void 7759 mcx_media_add_types(struct mcx_softc *sc) 7760 { 7761 struct mcx_reg_ptys ptys; 7762 int i; 7763 uint32_t proto_cap; 7764 7765 memset(&ptys, 0, sizeof(ptys)); 7766 ptys.rp_local_port = 1; 7767 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7768 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7769 sizeof(ptys)) != 0) { 7770 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7771 return; 7772 } 7773 7774 proto_cap = betoh32(ptys.rp_eth_proto_cap); 7775 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7776 const struct mcx_eth_proto_capability *cap; 7777 if (!ISSET(proto_cap, 1 << i)) 7778 continue; 7779 7780 cap = &mcx_eth_cap_map[i]; 7781 if (cap->cap_media == 0) 7782 continue; 7783 7784 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 7785 } 7786 } 7787 7788 static void 7789 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 7790 { 7791 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7792 struct mcx_reg_ptys ptys; 7793 int i; 7794 uint32_t proto_oper; 7795 uint64_t media_oper; 7796 7797 memset(&ptys, 0, sizeof(ptys)); 7798 ptys.rp_local_port = 1; 7799 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7800 7801 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7802 sizeof(ptys)) != 0) { 7803 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7804 return; 7805 } 7806 7807 proto_oper = betoh32(ptys.rp_eth_proto_oper); 7808 7809 media_oper = 0; 7810 7811 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7812 const struct mcx_eth_proto_capability *cap; 7813 if (!ISSET(proto_oper, 1 << i)) 7814 continue; 7815 7816 cap = &mcx_eth_cap_map[i]; 7817 7818 if (cap->cap_media != 0) 7819 media_oper = cap->cap_media; 7820 } 7821 7822 ifmr->ifm_status = IFM_AVALID; 7823 if (proto_oper != 0) { 7824 ifmr->ifm_status |= IFM_ACTIVE; 7825 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 7826 /* txpause, rxpause, duplex? */ 7827 } 7828 } 7829 7830 static int 7831 mcx_media_change(struct ifnet *ifp) 7832 { 7833 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7834 struct mcx_reg_ptys ptys; 7835 struct mcx_reg_paos paos; 7836 uint32_t media; 7837 int i, error; 7838 7839 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 7840 return EINVAL; 7841 7842 error = 0; 7843 7844 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 7845 /* read ptys to get supported media */ 7846 memset(&ptys, 0, sizeof(ptys)); 7847 ptys.rp_local_port = 1; 7848 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7849 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 7850 &ptys, sizeof(ptys)) != 0) { 7851 printf("%s: unable to read port type/speed\n", 7852 DEVNAME(sc)); 7853 return EIO; 7854 } 7855 7856 media = betoh32(ptys.rp_eth_proto_cap); 7857 } else { 7858 /* map media type */ 7859 media = 0; 7860 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7861 const struct mcx_eth_proto_capability *cap; 7862 7863 cap = &mcx_eth_cap_map[i]; 7864 if (cap->cap_media == 7865 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 7866 media = (1 << i); 7867 break; 7868 } 7869 } 7870 } 7871 7872 /* disable the port */ 7873 memset(&paos, 0, sizeof(paos)); 7874 paos.rp_local_port = 1; 7875 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 7876 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7877 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7878 sizeof(paos)) != 0) { 7879 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 7880 return EIO; 7881 } 7882 7883 memset(&ptys, 0, sizeof(ptys)); 7884 ptys.rp_local_port = 1; 7885 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7886 ptys.rp_eth_proto_admin = htobe32(media); 7887 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 7888 sizeof(ptys)) != 0) { 7889 printf("%s: unable to set port media type/speed\n", 7890 DEVNAME(sc)); 7891 error = EIO; 7892 } 7893 7894 /* re-enable the port to start negotiation */ 7895 memset(&paos, 0, sizeof(paos)); 7896 paos.rp_local_port = 1; 7897 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 7898 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7899 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7900 sizeof(paos)) != 0) { 7901 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 7902 error = EIO; 7903 } 7904 7905 return error; 7906 } 7907 7908 static void 7909 mcx_port_change(void *xsc) 7910 { 7911 struct mcx_softc *sc = xsc; 7912 struct ifnet *ifp = &sc->sc_ac.ac_if; 7913 struct mcx_reg_ptys ptys = { 7914 .rp_local_port = 1, 7915 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 7916 }; 7917 int link_state = LINK_STATE_DOWN; 7918 7919 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7920 sizeof(ptys)) == 0) { 7921 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 7922 uint64_t baudrate = 0; 7923 unsigned int i; 7924 7925 if (proto_oper != 0) 7926 link_state = LINK_STATE_FULL_DUPLEX; 7927 7928 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7929 const struct mcx_eth_proto_capability *cap; 7930 if (!ISSET(proto_oper, 1 << i)) 7931 continue; 7932 7933 cap = &mcx_eth_cap_map[i]; 7934 if (cap->cap_baudrate == 0) 7935 continue; 7936 7937 baudrate = cap->cap_baudrate; 7938 break; 7939 } 7940 7941 ifp->if_baudrate = baudrate; 7942 } 7943 7944 if (link_state != ifp->if_link_state) { 7945 ifp->if_link_state = link_state; 7946 if_link_state_change(ifp); 7947 } 7948 } 7949 7950 static inline uint32_t 7951 mcx_rd(struct mcx_softc *sc, bus_size_t r) 7952 { 7953 uint32_t word; 7954 7955 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 7956 7957 return (betoh32(word)); 7958 } 7959 7960 static inline void 7961 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 7962 { 7963 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 7964 } 7965 7966 static inline void 7967 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 7968 { 7969 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 7970 } 7971 7972 static uint64_t 7973 mcx_timer(struct mcx_softc *sc) 7974 { 7975 uint32_t hi, lo, ni; 7976 7977 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7978 for (;;) { 7979 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 7980 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 7981 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7982 7983 if (ni == hi) 7984 break; 7985 7986 hi = ni; 7987 } 7988 7989 return (((uint64_t)hi << 32) | (uint64_t)lo); 7990 } 7991 7992 static int 7993 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 7994 bus_size_t size, u_int align) 7995 { 7996 mxm->mxm_size = size; 7997 7998 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 7999 mxm->mxm_size, 0, 8000 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 8001 &mxm->mxm_map) != 0) 8002 return (1); 8003 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 8004 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 8005 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 8006 goto destroy; 8007 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 8008 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 8009 goto free; 8010 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 8011 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 8012 goto unmap; 8013 8014 return (0); 8015 unmap: 8016 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8017 free: 8018 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8019 destroy: 8020 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8021 return (1); 8022 } 8023 8024 static void 8025 mcx_dmamem_zero(struct mcx_dmamem *mxm) 8026 { 8027 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 8028 } 8029 8030 static void 8031 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 8032 { 8033 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 8034 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8035 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8036 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8037 } 8038 8039 static int 8040 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 8041 { 8042 bus_dma_segment_t *segs; 8043 bus_size_t len = pages * MCX_PAGE_SIZE; 8044 size_t seglen; 8045 8046 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 8047 if (segs == NULL) 8048 return (-1); 8049 8050 seglen = sizeof(*segs) * pages; 8051 8052 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 8053 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 8054 goto free_segs; 8055 8056 if (mhm->mhm_seg_count < pages) { 8057 size_t nseglen; 8058 8059 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 8060 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 8061 if (mhm->mhm_segs == NULL) 8062 goto free_dmamem; 8063 8064 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 8065 8066 memcpy(mhm->mhm_segs, segs, nseglen); 8067 8068 free(segs, M_DEVBUF, seglen); 8069 8070 segs = mhm->mhm_segs; 8071 seglen = nseglen; 8072 } else 8073 mhm->mhm_segs = segs; 8074 8075 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 8076 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 8077 &mhm->mhm_map) != 0) 8078 goto free_dmamem; 8079 8080 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 8081 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 8082 goto destroy; 8083 8084 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8085 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 8086 8087 mhm->mhm_npages = pages; 8088 8089 return (0); 8090 8091 destroy: 8092 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8093 free_dmamem: 8094 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8095 free_segs: 8096 free(segs, M_DEVBUF, seglen); 8097 mhm->mhm_segs = NULL; 8098 8099 return (-1); 8100 } 8101 8102 static void 8103 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 8104 { 8105 if (mhm->mhm_npages == 0) 8106 return; 8107 8108 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8109 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 8110 8111 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 8112 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8113 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8114 free(mhm->mhm_segs, M_DEVBUF, 8115 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 8116 8117 mhm->mhm_npages = 0; 8118 } 8119 8120 #if NKSTAT > 0 8121 struct mcx_ppcnt { 8122 char name[KSTAT_KV_NAMELEN]; 8123 enum kstat_kv_unit unit; 8124 }; 8125 8126 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 8127 { "Good Tx", KSTAT_KV_U_PACKETS, }, 8128 { "Good Rx", KSTAT_KV_U_PACKETS, }, 8129 { "FCS errs", KSTAT_KV_U_PACKETS, }, 8130 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8131 { "Good Tx", KSTAT_KV_U_BYTES, }, 8132 { "Good Rx", KSTAT_KV_U_BYTES, }, 8133 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 8134 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 8135 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 8136 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 8137 { "In Range Len", KSTAT_KV_U_PACKETS, }, 8138 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 8139 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 8140 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 8141 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 8142 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 8143 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 8144 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8145 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8146 }; 8147 CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 8148 8149 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 8150 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 8151 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 8152 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 8153 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 8154 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 8155 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 8156 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 8157 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 8158 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 8159 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 8160 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 8161 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 8162 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 8163 }; 8164 CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 8165 8166 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 8167 { "Drop Events", KSTAT_KV_U_PACKETS, }, 8168 { "Octets", KSTAT_KV_U_BYTES, }, 8169 { "Packets", KSTAT_KV_U_PACKETS, }, 8170 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 8171 { "Multicasts", KSTAT_KV_U_PACKETS, }, 8172 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 8173 { "Undersize", KSTAT_KV_U_PACKETS, }, 8174 { "Oversize", KSTAT_KV_U_PACKETS, }, 8175 { "Fragments", KSTAT_KV_U_PACKETS, }, 8176 { "Jabbers", KSTAT_KV_U_PACKETS, }, 8177 { "Collisions", KSTAT_KV_U_NONE, }, 8178 { "64B", KSTAT_KV_U_PACKETS, }, 8179 { "65-127B", KSTAT_KV_U_PACKETS, }, 8180 { "128-255B", KSTAT_KV_U_PACKETS, }, 8181 { "256-511B", KSTAT_KV_U_PACKETS, }, 8182 { "512-1023B", KSTAT_KV_U_PACKETS, }, 8183 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 8184 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 8185 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 8186 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 8187 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 8188 }; 8189 CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 8190 8191 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 8192 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8193 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 8194 { "Single Colls", KSTAT_KV_U_PACKETS, }, 8195 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 8196 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 8197 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 8198 { "Late Colls", KSTAT_KV_U_NONE, }, 8199 { "Exess Colls", KSTAT_KV_U_NONE, }, 8200 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 8201 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 8202 { "Too Long", KSTAT_KV_U_PACKETS, }, 8203 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 8204 { "Symbol Errs", KSTAT_KV_U_NONE, }, 8205 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 8206 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8207 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8208 }; 8209 CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 8210 8211 struct mcx_kstat_ppcnt { 8212 const char *ksp_name; 8213 const struct mcx_ppcnt *ksp_tpl; 8214 unsigned int ksp_n; 8215 uint8_t ksp_grp; 8216 }; 8217 8218 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 8219 .ksp_name = "ieee802.3", 8220 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 8221 .ksp_n = nitems(mcx_ppcnt_ieee8023_tpl), 8222 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 8223 }; 8224 8225 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 8226 .ksp_name = "rfc2863", 8227 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 8228 .ksp_n = nitems(mcx_ppcnt_rfc2863_tpl), 8229 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 8230 }; 8231 8232 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 8233 .ksp_name = "rfc2819", 8234 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 8235 .ksp_n = nitems(mcx_ppcnt_rfc2819_tpl), 8236 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 8237 }; 8238 8239 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 8240 .ksp_name = "rfc3635", 8241 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 8242 .ksp_n = nitems(mcx_ppcnt_rfc3635_tpl), 8243 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 8244 }; 8245 8246 static int mcx_kstat_ppcnt_read(struct kstat *); 8247 8248 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 8249 static void mcx_kstat_attach_queues(struct mcx_softc *sc); 8250 8251 static struct kstat * 8252 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 8253 const struct mcx_kstat_ppcnt *ksp) 8254 { 8255 struct kstat *ks; 8256 struct kstat_kv *kvs; 8257 unsigned int i; 8258 8259 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 8260 if (ks == NULL) 8261 return (NULL); 8262 8263 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 8264 M_DEVBUF, M_WAITOK); 8265 8266 for (i = 0; i < ksp->ksp_n; i++) { 8267 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 8268 8269 kstat_kv_unit_init(&kvs[i], tpl->name, 8270 KSTAT_KV_T_COUNTER64, tpl->unit); 8271 } 8272 8273 ks->ks_softc = sc; 8274 ks->ks_ptr = (void *)ksp; 8275 ks->ks_data = kvs; 8276 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 8277 ks->ks_read = mcx_kstat_ppcnt_read; 8278 8279 kstat_install(ks); 8280 8281 return (ks); 8282 } 8283 8284 static void 8285 mcx_kstat_attach(struct mcx_softc *sc) 8286 { 8287 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 8288 &mcx_kstat_ppcnt_ieee8023); 8289 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 8290 &mcx_kstat_ppcnt_rfc2863); 8291 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 8292 &mcx_kstat_ppcnt_rfc2819); 8293 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 8294 &mcx_kstat_ppcnt_rfc3635); 8295 8296 mcx_kstat_attach_tmps(sc); 8297 mcx_kstat_attach_queues(sc); 8298 } 8299 8300 static int 8301 mcx_kstat_ppcnt_read(struct kstat *ks) 8302 { 8303 struct mcx_softc *sc = ks->ks_softc; 8304 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 8305 struct mcx_reg_ppcnt ppcnt = { 8306 .ppcnt_grp = ksp->ksp_grp, 8307 .ppcnt_local_port = 1, 8308 }; 8309 struct kstat_kv *kvs = ks->ks_data; 8310 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 8311 unsigned int i; 8312 int rv; 8313 8314 KERNEL_LOCK(); /* XXX */ 8315 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 8316 &ppcnt, sizeof(ppcnt)); 8317 KERNEL_UNLOCK(); 8318 if (rv != 0) 8319 return (EIO); 8320 8321 nanouptime(&ks->ks_updated); 8322 8323 for (i = 0; i < ksp->ksp_n; i++) 8324 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 8325 8326 return (0); 8327 } 8328 8329 struct mcx_kstat_mtmp { 8330 struct kstat_kv ktmp_name; 8331 struct kstat_kv ktmp_temperature; 8332 struct kstat_kv ktmp_threshold_lo; 8333 struct kstat_kv ktmp_threshold_hi; 8334 }; 8335 8336 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 8337 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 8338 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 8339 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 8340 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 8341 }; 8342 8343 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 8344 8345 static int mcx_kstat_mtmp_read(struct kstat *); 8346 8347 static void 8348 mcx_kstat_attach_tmps(struct mcx_softc *sc) 8349 { 8350 struct kstat *ks; 8351 struct mcx_reg_mtcap mtcap; 8352 struct mcx_kstat_mtmp *ktmp; 8353 uint64_t map; 8354 unsigned int i, n; 8355 8356 memset(&mtcap, 0, sizeof(mtcap)); 8357 8358 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8359 &mtcap, sizeof(mtcap)) != 0) { 8360 /* unable to find temperature sensors */ 8361 return; 8362 } 8363 8364 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8365 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8366 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8367 8368 n = 0; 8369 map = bemtoh64(&mtcap.mtcap_sensor_map); 8370 for (i = 0; i < sizeof(map) * NBBY; i++) { 8371 if (!ISSET(map, (1ULL << i))) 8372 continue; 8373 8374 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8375 KSTAT_T_KV, 0); 8376 if (ks == NULL) { 8377 /* unable to attach temperature sensor %u, i */ 8378 continue; 8379 } 8380 8381 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8382 *ktmp = mcx_kstat_mtmp_tpl; 8383 8384 ks->ks_data = ktmp; 8385 ks->ks_datalen = sizeof(*ktmp); 8386 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8387 ks->ks_read = mcx_kstat_mtmp_read; 8388 8389 ks->ks_softc = sc; 8390 kstat_install(ks); 8391 8392 sc->sc_kstat_mtmp[n++] = ks; 8393 if (n >= sc->sc_kstat_mtmp_count) 8394 break; 8395 } 8396 } 8397 8398 static uint64_t 8399 mcx_tmp_to_uK(uint16_t *t) 8400 { 8401 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8402 mt *= 1000000 / 8; /* convert to uC */ 8403 mt += 273150000; /* convert to uK */ 8404 8405 return (mt); 8406 } 8407 8408 static int 8409 mcx_kstat_mtmp_read(struct kstat *ks) 8410 { 8411 struct mcx_softc *sc = ks->ks_softc; 8412 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8413 struct mcx_reg_mtmp mtmp; 8414 int rv; 8415 struct timeval updated; 8416 8417 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8418 8419 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8420 return (0); 8421 8422 memset(&mtmp, 0, sizeof(mtmp)); 8423 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8424 8425 KERNEL_LOCK(); /* XXX */ 8426 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8427 &mtmp, sizeof(mtmp)); 8428 KERNEL_UNLOCK(); 8429 if (rv != 0) 8430 return (EIO); 8431 8432 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8433 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8434 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8435 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8436 kstat_kv_temp(&ktmp->ktmp_temperature) = 8437 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8438 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8439 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8440 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8441 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8442 8443 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8444 8445 return (0); 8446 } 8447 8448 struct mcx_queuestat { 8449 char name[KSTAT_KV_NAMELEN]; 8450 enum kstat_kv_type type; 8451 }; 8452 8453 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = { 8454 { "RQ SW prod", KSTAT_KV_T_COUNTER64 }, 8455 { "RQ HW prod", KSTAT_KV_T_COUNTER64 }, 8456 { "RQ HW cons", KSTAT_KV_T_COUNTER64 }, 8457 { "RQ HW state", KSTAT_KV_T_ISTR }, 8458 8459 { "SQ SW prod", KSTAT_KV_T_COUNTER64 }, 8460 { "SQ SW cons", KSTAT_KV_T_COUNTER64 }, 8461 { "SQ HW prod", KSTAT_KV_T_COUNTER64 }, 8462 { "SQ HW cons", KSTAT_KV_T_COUNTER64 }, 8463 { "SQ HW state", KSTAT_KV_T_ISTR }, 8464 8465 { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, 8466 { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, 8467 { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, 8468 { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, 8469 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, 8470 { "CQ HW status", KSTAT_KV_T_ISTR }, 8471 { "CQ HW state", KSTAT_KV_T_ISTR }, 8472 8473 { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, 8474 { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, 8475 { "EQ HW cons", KSTAT_KV_T_COUNTER64 }, 8476 { "EQ HW status", KSTAT_KV_T_ISTR }, 8477 { "EQ HW state", KSTAT_KV_T_ISTR }, 8478 }; 8479 8480 static int mcx_kstat_queue_read(struct kstat *); 8481 8482 static void 8483 mcx_kstat_attach_queues(struct mcx_softc *sc) 8484 { 8485 struct kstat *ks; 8486 struct kstat_kv *kvs; 8487 int q, i; 8488 8489 for (q = 0; q < sc->sc_nqueues; q++) { 8490 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, 8491 KSTAT_T_KV, 0); 8492 if (ks == NULL) { 8493 /* unable to attach queue stats %u, q */ 8494 continue; 8495 } 8496 8497 kvs = mallocarray(nitems(mcx_queue_kstat_tpl), 8498 sizeof(*kvs), M_DEVBUF, M_WAITOK); 8499 8500 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) { 8501 const struct mcx_queuestat *tpl = 8502 &mcx_queue_kstat_tpl[i]; 8503 8504 kstat_kv_init(&kvs[i], tpl->name, tpl->type); 8505 } 8506 8507 ks->ks_softc = &sc->sc_queues[q]; 8508 ks->ks_data = kvs; 8509 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs); 8510 ks->ks_read = mcx_kstat_queue_read; 8511 8512 sc->sc_queues[q].q_kstat = ks; 8513 kstat_install(ks); 8514 } 8515 } 8516 8517 static int 8518 mcx_kstat_queue_read(struct kstat *ks) 8519 { 8520 struct mcx_queues *q = ks->ks_softc; 8521 struct mcx_softc *sc = q->q_sc; 8522 struct kstat_kv *kvs = ks->ks_data; 8523 union { 8524 struct mcx_rq_ctx rq; 8525 struct mcx_sq_ctx sq; 8526 struct mcx_cq_ctx cq; 8527 struct mcx_eq_ctx eq; 8528 } u; 8529 const char *text; 8530 int error = 0; 8531 8532 KERNEL_LOCK(); 8533 8534 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) { 8535 error = EIO; 8536 goto out; 8537 } 8538 8539 kstat_kv_u64(kvs++) = q->q_rx.rx_prod; 8540 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter); 8541 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter); 8542 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >> 8543 MCX_RQ_CTX_STATE_SHIFT) { 8544 case MCX_RQ_CTX_STATE_RST: 8545 text = "RST"; 8546 break; 8547 case MCX_RQ_CTX_STATE_RDY: 8548 text = "RDY"; 8549 break; 8550 case MCX_RQ_CTX_STATE_ERR: 8551 text = "ERR"; 8552 break; 8553 default: 8554 text = "unknown"; 8555 break; 8556 } 8557 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8558 kvs++; 8559 8560 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) { 8561 error = EIO; 8562 goto out; 8563 } 8564 8565 kstat_kv_u64(kvs++) = q->q_tx.tx_prod; 8566 kstat_kv_u64(kvs++) = q->q_tx.tx_cons; 8567 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter); 8568 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter); 8569 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >> 8570 MCX_SQ_CTX_STATE_SHIFT) { 8571 case MCX_SQ_CTX_STATE_RST: 8572 text = "RST"; 8573 break; 8574 case MCX_SQ_CTX_STATE_RDY: 8575 text = "RDY"; 8576 break; 8577 case MCX_SQ_CTX_STATE_ERR: 8578 text = "ERR"; 8579 break; 8580 default: 8581 text = "unknown"; 8582 break; 8583 } 8584 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8585 kvs++; 8586 8587 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) { 8588 error = EIO; 8589 goto out; 8590 } 8591 8592 kstat_kv_u64(kvs++) = q->q_cq.cq_cons; 8593 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); 8594 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); 8595 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); 8596 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); 8597 8598 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >> 8599 MCX_CQ_CTX_STATUS_SHIFT) { 8600 case MCX_CQ_CTX_STATUS_OK: 8601 text = "OK"; 8602 break; 8603 case MCX_CQ_CTX_STATUS_OVERFLOW: 8604 text = "overflow"; 8605 break; 8606 case MCX_CQ_CTX_STATUS_WRITE_FAIL: 8607 text = "write fail"; 8608 break; 8609 default: 8610 text = "unknown"; 8611 break; 8612 } 8613 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8614 kvs++; 8615 8616 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >> 8617 MCX_CQ_CTX_STATE_SHIFT) { 8618 case MCX_CQ_CTX_STATE_SOLICITED: 8619 text = "solicited"; 8620 break; 8621 case MCX_CQ_CTX_STATE_ARMED: 8622 text = "armed"; 8623 break; 8624 case MCX_CQ_CTX_STATE_FIRED: 8625 text = "fired"; 8626 break; 8627 default: 8628 text = "unknown"; 8629 break; 8630 } 8631 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8632 kvs++; 8633 8634 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { 8635 error = EIO; 8636 goto out; 8637 } 8638 8639 kstat_kv_u64(kvs++) = q->q_eq.eq_cons; 8640 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter); 8641 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter); 8642 8643 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >> 8644 MCX_EQ_CTX_STATUS_SHIFT) { 8645 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE: 8646 text = "write fail"; 8647 break; 8648 case MCX_EQ_CTX_STATUS_OK: 8649 text = "OK"; 8650 break; 8651 default: 8652 text = "unknown"; 8653 break; 8654 } 8655 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8656 kvs++; 8657 8658 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >> 8659 MCX_EQ_CTX_STATE_SHIFT) { 8660 case MCX_EQ_CTX_STATE_ARMED: 8661 text = "armed"; 8662 break; 8663 case MCX_EQ_CTX_STATE_FIRED: 8664 text = "fired"; 8665 break; 8666 default: 8667 text = "unknown"; 8668 break; 8669 } 8670 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8671 kvs++; 8672 8673 nanouptime(&ks->ks_updated); 8674 out: 8675 KERNEL_UNLOCK(); 8676 return (error); 8677 } 8678 8679 #endif /* NKSTAT > 0 */ 8680 8681 static unsigned int 8682 mcx_timecounter_read(struct timecounter *tc) 8683 { 8684 struct mcx_softc *sc = tc->tc_priv; 8685 8686 return (mcx_rd(sc, MCX_INTERNAL_TIMER_L)); 8687 } 8688 8689 static void 8690 mcx_timecounter_attach(struct mcx_softc *sc) 8691 { 8692 struct timecounter *tc = &sc->sc_timecounter; 8693 8694 tc->tc_get_timecount = mcx_timecounter_read; 8695 tc->tc_counter_mask = ~0U; 8696 tc->tc_frequency = sc->sc_khz * 1000; 8697 tc->tc_name = sc->sc_dev.dv_xname; 8698 tc->tc_quality = -100; 8699 tc->tc_priv = sc; 8700 8701 tc_init(tc); 8702 } 8703