1 /* $NetBSD: if_mcx.c,v 1.22 2021/09/26 20:14:07 jmcneill Exp $ */ 2 /* $OpenBSD: if_mcx.c,v 1.101 2021/06/02 19:16:11 patrick Exp $ */ 3 4 /* 5 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 6 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #ifdef _KERNEL_OPT 22 #include "opt_net_mpsafe.h" 23 #endif 24 25 #include <sys/cdefs.h> 26 __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.22 2021/09/26 20:14:07 jmcneill Exp $"); 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/sockio.h> 31 #include <sys/mbuf.h> 32 #include <sys/kernel.h> 33 #include <sys/socket.h> 34 #include <sys/device.h> 35 #include <sys/pool.h> 36 #include <sys/queue.h> 37 #include <sys/callout.h> 38 #include <sys/workqueue.h> 39 #include <sys/atomic.h> 40 #include <sys/timetc.h> 41 #include <sys/kmem.h> 42 #include <sys/bus.h> 43 #include <sys/interrupt.h> 44 #include <sys/pcq.h> 45 #include <sys/cpu.h> 46 47 #include <machine/intr.h> 48 49 #include <net/if.h> 50 #include <net/if_dl.h> 51 #include <net/if_ether.h> 52 #include <net/if_media.h> 53 #include <net/if_vlanvar.h> 54 #include <net/toeplitz.h> 55 56 #include <net/bpf.h> 57 58 #include <netinet/in.h> 59 60 #include <dev/pci/pcireg.h> 61 #include <dev/pci/pcivar.h> 62 #include <dev/pci/pcidevs.h> 63 64 /* TODO: Port kstat key/value stuff to evcnt/sysmon */ 65 #define NKSTAT 0 66 67 /* XXX This driver is not yet MP-safe; don't claim to be! */ 68 /* #ifdef NET_MPSAFE */ 69 /* #define MCX_MPSAFE 1 */ 70 /* #define CALLOUT_FLAGS CALLOUT_MPSAFE */ 71 /* #else */ 72 #define CALLOUT_FLAGS 0 73 /* #endif */ 74 75 #define MCX_TXQ_NUM 2048 76 77 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 78 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 79 80 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 81 82 #define MCX_FW_VER 0x0000 83 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 84 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 85 #define MCX_CMDIF_FW_SUBVER 0x0004 86 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 87 #define MCX_CMDIF(_v) ((_v) >> 16) 88 89 #define MCX_ISSI 1 /* as per the PRM */ 90 #define MCX_CMD_IF_SUPPORTED 5 91 92 #define MCX_HARDMTU 9500 93 94 #define MCX_PAGE_SHIFT 12 95 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 96 97 /* queue sizes */ 98 #define MCX_LOG_EQ_SIZE 7 99 #define MCX_LOG_CQ_SIZE 12 100 #define MCX_LOG_RQ_SIZE 10 101 #define MCX_LOG_SQ_SIZE 11 102 103 #define MCX_MAX_QUEUES 16 104 105 /* completion event moderation - about 10khz, or 90% of the cq */ 106 #define MCX_CQ_MOD_PERIOD 50 107 #define MCX_CQ_MOD_COUNTER \ 108 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 109 110 #define MCX_LOG_SQ_ENTRY_SIZE 6 111 #define MCX_SQ_ENTRY_MAX_SLOTS 4 112 #define MCX_SQ_SEGS_PER_SLOT \ 113 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 114 #define MCX_SQ_MAX_SEGMENTS \ 115 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 116 117 #define MCX_LOG_FLOW_TABLE_SIZE 5 118 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 119 #define MCX_NUM_MCAST_FLOWS \ 120 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 121 122 #define MCX_SQ_INLINE_SIZE 18 123 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 124 125 /* doorbell offsets */ 126 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 127 128 #define MCX_CQ_DOORBELL_BASE 0 129 #define MCX_CQ_DOORBELL_STRIDE 64 130 131 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 132 #define MCX_WQ_DOORBELL_STRIDE 64 133 /* make sure the doorbells fit */ 134 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 135 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 136 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 137 138 #define MCX_WQ_DOORBELL_MASK 0xffff 139 140 /* uar registers */ 141 #define MCX_UAR_CQ_DOORBELL 0x20 142 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 143 #define MCX_UAR_EQ_DOORBELL 0x48 144 #define MCX_UAR_BF 0x800 145 146 #define MCX_CMDQ_ADDR_HI 0x0010 147 #define MCX_CMDQ_ADDR_LO 0x0014 148 #define MCX_CMDQ_ADDR_NMASK 0xfff 149 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 150 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 151 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 152 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 153 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 154 155 #define MCX_CMDQ_DOORBELL 0x0018 156 157 #define MCX_STATE 0x01fc 158 #define MCX_STATE_MASK (1U << 31) 159 #define MCX_STATE_INITIALIZING (1 << 31) 160 #define MCX_STATE_READY (0 << 31) 161 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 162 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 163 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 164 165 #define MCX_INTERNAL_TIMER 0x1000 166 #define MCX_INTERNAL_TIMER_H 0x1000 167 #define MCX_INTERNAL_TIMER_L 0x1004 168 169 #define MCX_CLEAR_INT 0x100c 170 171 #define MCX_REG_OP_WRITE 0 172 #define MCX_REG_OP_READ 1 173 174 #define MCX_REG_PMLP 0x5002 175 #define MCX_REG_PMTU 0x5003 176 #define MCX_REG_PTYS 0x5004 177 #define MCX_REG_PAOS 0x5006 178 #define MCX_REG_PFCC 0x5007 179 #define MCX_REG_PPCNT 0x5008 180 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 181 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 182 #define MCX_REG_MCIA 0x9014 183 #define MCX_REG_MCAM 0x907f 184 185 #define MCX_ETHER_CAP_SGMII 0 186 #define MCX_ETHER_CAP_1000_KX 1 187 #define MCX_ETHER_CAP_10G_CX4 2 188 #define MCX_ETHER_CAP_10G_KX4 3 189 #define MCX_ETHER_CAP_10G_KR 4 190 #define MCX_ETHER_CAP_20G_KR2 5 191 #define MCX_ETHER_CAP_40G_CR4 6 192 #define MCX_ETHER_CAP_40G_KR4 7 193 #define MCX_ETHER_CAP_56G_R4 8 194 #define MCX_ETHER_CAP_10G_CR 12 195 #define MCX_ETHER_CAP_10G_SR 13 196 #define MCX_ETHER_CAP_10G_LR 14 197 #define MCX_ETHER_CAP_40G_SR4 15 198 #define MCX_ETHER_CAP_40G_LR4 16 199 #define MCX_ETHER_CAP_50G_SR2 18 200 #define MCX_ETHER_CAP_100G_CR4 20 201 #define MCX_ETHER_CAP_100G_SR4 21 202 #define MCX_ETHER_CAP_100G_KR4 22 203 #define MCX_ETHER_CAP_100G_LR4 23 204 #define MCX_ETHER_CAP_100_TX 24 205 #define MCX_ETHER_CAP_1000_T 25 206 #define MCX_ETHER_CAP_10G_T 26 207 #define MCX_ETHER_CAP_25G_CR 27 208 #define MCX_ETHER_CAP_25G_KR 28 209 #define MCX_ETHER_CAP_25G_SR 29 210 #define MCX_ETHER_CAP_50G_CR2 30 211 #define MCX_ETHER_CAP_50G_KR2 31 212 213 #define MCX_MAX_CQE 32 214 215 #define MCX_CMD_QUERY_HCA_CAP 0x100 216 #define MCX_CMD_QUERY_ADAPTER 0x101 217 #define MCX_CMD_INIT_HCA 0x102 218 #define MCX_CMD_TEARDOWN_HCA 0x103 219 #define MCX_CMD_ENABLE_HCA 0x104 220 #define MCX_CMD_DISABLE_HCA 0x105 221 #define MCX_CMD_QUERY_PAGES 0x107 222 #define MCX_CMD_MANAGE_PAGES 0x108 223 #define MCX_CMD_SET_HCA_CAP 0x109 224 #define MCX_CMD_QUERY_ISSI 0x10a 225 #define MCX_CMD_SET_ISSI 0x10b 226 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 227 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 228 #define MCX_CMD_CREATE_EQ 0x301 229 #define MCX_CMD_DESTROY_EQ 0x302 230 #define MCX_CMD_QUERY_EQ 0x303 231 #define MCX_CMD_CREATE_CQ 0x400 232 #define MCX_CMD_DESTROY_CQ 0x401 233 #define MCX_CMD_QUERY_CQ 0x402 234 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 235 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 236 0x755 237 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 238 #define MCX_CMD_ALLOC_PD 0x800 239 #define MCX_CMD_ALLOC_UAR 0x802 240 #define MCX_CMD_ACCESS_REG 0x805 241 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 242 #define MCX_CMD_CREATE_TIR 0x900 243 #define MCX_CMD_DESTROY_TIR 0x902 244 #define MCX_CMD_CREATE_SQ 0x904 245 #define MCX_CMD_MODIFY_SQ 0x905 246 #define MCX_CMD_DESTROY_SQ 0x906 247 #define MCX_CMD_QUERY_SQ 0x907 248 #define MCX_CMD_CREATE_RQ 0x908 249 #define MCX_CMD_MODIFY_RQ 0x909 250 #define MCX_CMD_DESTROY_RQ 0x90a 251 #define MCX_CMD_QUERY_RQ 0x90b 252 #define MCX_CMD_CREATE_TIS 0x912 253 #define MCX_CMD_DESTROY_TIS 0x914 254 #define MCX_CMD_CREATE_RQT 0x916 255 #define MCX_CMD_DESTROY_RQT 0x918 256 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 257 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 258 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 259 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 260 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 261 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 262 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 263 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 264 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 265 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 266 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 267 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 268 269 #define MCX_QUEUE_STATE_RST 0 270 #define MCX_QUEUE_STATE_RDY 1 271 #define MCX_QUEUE_STATE_ERR 3 272 273 #define MCX_FLOW_TABLE_TYPE_RX 0 274 #define MCX_FLOW_TABLE_TYPE_TX 1 275 276 #define MCX_CMDQ_INLINE_DATASIZE 16 277 278 struct mcx_cmdq_entry { 279 uint8_t cq_type; 280 #define MCX_CMDQ_TYPE_PCIE 0x7 281 uint8_t cq_reserved0[3]; 282 283 uint32_t cq_input_length; 284 uint64_t cq_input_ptr; 285 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 286 287 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 288 uint64_t cq_output_ptr; 289 uint32_t cq_output_length; 290 291 uint8_t cq_token; 292 uint8_t cq_signature; 293 uint8_t cq_reserved1[1]; 294 uint8_t cq_status; 295 #define MCX_CQ_STATUS_SHIFT 1 296 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 297 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 298 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 299 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 300 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 301 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 302 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 303 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 304 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 305 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 306 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 307 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 308 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 309 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 310 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 311 (0x10 << MCX_CQ_STATUS_SHIFT) 312 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 313 #define MCX_CQ_STATUS_OWN_MASK 0x1 314 #define MCX_CQ_STATUS_OWN_SW 0x0 315 #define MCX_CQ_STATUS_OWN_HW 0x1 316 } __packed __aligned(8); 317 318 #define MCX_CMDQ_MAILBOX_DATASIZE 512 319 320 struct mcx_cmdq_mailbox { 321 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 322 uint8_t mb_reserved0[48]; 323 uint64_t mb_next_ptr; 324 uint32_t mb_block_number; 325 uint8_t mb_reserved1[1]; 326 uint8_t mb_token; 327 uint8_t mb_ctrl_signature; 328 uint8_t mb_signature; 329 } __packed __aligned(8); 330 331 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 332 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 333 MCX_CMDQ_MAILBOX_ALIGN) 334 /* 335 * command mailbox structures 336 */ 337 338 struct mcx_cmd_enable_hca_in { 339 uint16_t cmd_opcode; 340 uint8_t cmd_reserved0[4]; 341 uint16_t cmd_op_mod; 342 uint8_t cmd_reserved1[2]; 343 uint16_t cmd_function_id; 344 uint8_t cmd_reserved2[4]; 345 } __packed __aligned(4); 346 347 struct mcx_cmd_enable_hca_out { 348 uint8_t cmd_status; 349 uint8_t cmd_reserved0[3]; 350 uint32_t cmd_syndrome; 351 uint8_t cmd_reserved1[4]; 352 } __packed __aligned(4); 353 354 struct mcx_cmd_init_hca_in { 355 uint16_t cmd_opcode; 356 uint8_t cmd_reserved0[4]; 357 uint16_t cmd_op_mod; 358 uint8_t cmd_reserved1[8]; 359 } __packed __aligned(4); 360 361 struct mcx_cmd_init_hca_out { 362 uint8_t cmd_status; 363 uint8_t cmd_reserved0[3]; 364 uint32_t cmd_syndrome; 365 uint8_t cmd_reserved1[8]; 366 } __packed __aligned(4); 367 368 struct mcx_cmd_teardown_hca_in { 369 uint16_t cmd_opcode; 370 uint8_t cmd_reserved0[4]; 371 uint16_t cmd_op_mod; 372 uint8_t cmd_reserved1[2]; 373 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 374 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 375 uint16_t cmd_profile; 376 uint8_t cmd_reserved2[4]; 377 } __packed __aligned(4); 378 379 struct mcx_cmd_teardown_hca_out { 380 uint8_t cmd_status; 381 uint8_t cmd_reserved0[3]; 382 uint32_t cmd_syndrome; 383 uint8_t cmd_reserved1[8]; 384 } __packed __aligned(4); 385 386 struct mcx_cmd_access_reg_in { 387 uint16_t cmd_opcode; 388 uint8_t cmd_reserved0[4]; 389 uint16_t cmd_op_mod; 390 uint8_t cmd_reserved1[2]; 391 uint16_t cmd_register_id; 392 uint32_t cmd_argument; 393 } __packed __aligned(4); 394 395 struct mcx_cmd_access_reg_out { 396 uint8_t cmd_status; 397 uint8_t cmd_reserved0[3]; 398 uint32_t cmd_syndrome; 399 uint8_t cmd_reserved1[8]; 400 } __packed __aligned(4); 401 402 struct mcx_reg_pmtu { 403 uint8_t rp_reserved1; 404 uint8_t rp_local_port; 405 uint8_t rp_reserved2[2]; 406 uint16_t rp_max_mtu; 407 uint8_t rp_reserved3[2]; 408 uint16_t rp_admin_mtu; 409 uint8_t rp_reserved4[2]; 410 uint16_t rp_oper_mtu; 411 uint8_t rp_reserved5[2]; 412 } __packed __aligned(4); 413 414 struct mcx_reg_ptys { 415 uint8_t rp_reserved1; 416 uint8_t rp_local_port; 417 uint8_t rp_reserved2; 418 uint8_t rp_proto_mask; 419 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 420 uint8_t rp_reserved3[8]; 421 uint32_t rp_eth_proto_cap; 422 uint8_t rp_reserved4[8]; 423 uint32_t rp_eth_proto_admin; 424 uint8_t rp_reserved5[8]; 425 uint32_t rp_eth_proto_oper; 426 uint8_t rp_reserved6[24]; 427 } __packed __aligned(4); 428 429 struct mcx_reg_paos { 430 uint8_t rp_reserved1; 431 uint8_t rp_local_port; 432 uint8_t rp_admin_status; 433 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 434 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 435 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 436 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 437 uint8_t rp_oper_status; 438 #define MCX_REG_PAOS_OPER_STATUS_UP 1 439 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 440 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 441 uint8_t rp_admin_state_update; 442 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 443 uint8_t rp_reserved2[11]; 444 } __packed __aligned(4); 445 446 struct mcx_reg_pfcc { 447 uint8_t rp_reserved1; 448 uint8_t rp_local_port; 449 uint8_t rp_reserved2[3]; 450 uint8_t rp_prio_mask_tx; 451 uint8_t rp_reserved3; 452 uint8_t rp_prio_mask_rx; 453 uint8_t rp_pptx_aptx; 454 uint8_t rp_pfctx; 455 uint8_t rp_fctx_dis; 456 uint8_t rp_reserved4; 457 uint8_t rp_pprx_aprx; 458 uint8_t rp_pfcrx; 459 uint8_t rp_reserved5[2]; 460 uint16_t rp_dev_stall_min; 461 uint16_t rp_dev_stall_crit; 462 uint8_t rp_reserved6[12]; 463 } __packed __aligned(4); 464 465 #define MCX_PMLP_MODULE_NUM_MASK 0xff 466 struct mcx_reg_pmlp { 467 uint8_t rp_rxtx; 468 uint8_t rp_local_port; 469 uint8_t rp_reserved0; 470 uint8_t rp_width; 471 uint32_t rp_lane0_mapping; 472 uint32_t rp_lane1_mapping; 473 uint32_t rp_lane2_mapping; 474 uint32_t rp_lane3_mapping; 475 uint8_t rp_reserved1[44]; 476 } __packed __aligned(4); 477 478 struct mcx_reg_ppcnt { 479 uint8_t ppcnt_swid; 480 uint8_t ppcnt_local_port; 481 uint8_t ppcnt_pnat; 482 uint8_t ppcnt_grp; 483 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 484 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 485 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 486 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 487 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 488 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 489 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 490 491 uint8_t ppcnt_clr; 492 uint8_t ppcnt_reserved1[2]; 493 uint8_t ppcnt_prio_tc; 494 #define MCX_REG_PPCNT_CLR (1 << 7) 495 496 uint8_t ppcnt_counter_set[248]; 497 } __packed __aligned(8); 498 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 499 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 500 sizeof(uint64_t)) == 0); 501 502 enum mcx_ppcnt_ieee8023 { 503 frames_transmitted_ok, 504 frames_received_ok, 505 frame_check_sequence_errors, 506 alignment_errors, 507 octets_transmitted_ok, 508 octets_received_ok, 509 multicast_frames_xmitted_ok, 510 broadcast_frames_xmitted_ok, 511 multicast_frames_received_ok, 512 broadcast_frames_received_ok, 513 in_range_length_errors, 514 out_of_range_length_field, 515 frame_too_long_errors, 516 symbol_error_during_carrier, 517 mac_control_frames_transmitted, 518 mac_control_frames_received, 519 unsupported_opcodes_received, 520 pause_mac_ctrl_frames_received, 521 pause_mac_ctrl_frames_transmitted, 522 523 mcx_ppcnt_ieee8023_count 524 }; 525 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 526 527 enum mcx_ppcnt_rfc2863 { 528 in_octets, 529 in_ucast_pkts, 530 in_discards, 531 in_errors, 532 in_unknown_protos, 533 out_octets, 534 out_ucast_pkts, 535 out_discards, 536 out_errors, 537 in_multicast_pkts, 538 in_broadcast_pkts, 539 out_multicast_pkts, 540 out_broadcast_pkts, 541 542 mcx_ppcnt_rfc2863_count 543 }; 544 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 545 546 enum mcx_ppcnt_rfc2819 { 547 drop_events, 548 octets, 549 pkts, 550 broadcast_pkts, 551 multicast_pkts, 552 crc_align_errors, 553 undersize_pkts, 554 oversize_pkts, 555 fragments, 556 jabbers, 557 collisions, 558 pkts64octets, 559 pkts65to127octets, 560 pkts128to255octets, 561 pkts256to511octets, 562 pkts512to1023octets, 563 pkts1024to1518octets, 564 pkts1519to2047octets, 565 pkts2048to4095octets, 566 pkts4096to8191octets, 567 pkts8192to10239octets, 568 569 mcx_ppcnt_rfc2819_count 570 }; 571 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 572 573 enum mcx_ppcnt_rfc3635 { 574 dot3stats_alignment_errors, 575 dot3stats_fcs_errors, 576 dot3stats_single_collision_frames, 577 dot3stats_multiple_collision_frames, 578 dot3stats_sqe_test_errors, 579 dot3stats_deferred_transmissions, 580 dot3stats_late_collisions, 581 dot3stats_excessive_collisions, 582 dot3stats_internal_mac_transmit_errors, 583 dot3stats_carrier_sense_errors, 584 dot3stats_frame_too_longs, 585 dot3stats_internal_mac_receive_errors, 586 dot3stats_symbol_errors, 587 dot3control_in_unknown_opcodes, 588 dot3in_pause_frames, 589 dot3out_pause_frames, 590 591 mcx_ppcnt_rfc3635_count 592 }; 593 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 594 595 struct mcx_reg_mcam { 596 uint8_t _reserved1[1]; 597 uint8_t mcam_feature_group; 598 uint8_t _reserved2[1]; 599 uint8_t mcam_access_reg_group; 600 uint8_t _reserved3[4]; 601 uint8_t mcam_access_reg_cap_mask[16]; 602 uint8_t _reserved4[16]; 603 uint8_t mcam_feature_cap_mask[16]; 604 uint8_t _reserved5[16]; 605 } __packed __aligned(4); 606 607 #define MCX_BITFIELD_BIT(bf, b) (bf[(sizeof bf - 1) - (b / 8)] & (b % 8)) 608 609 #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP 6 610 611 struct mcx_reg_mtcap { 612 uint8_t _reserved1[3]; 613 uint8_t mtcap_sensor_count; 614 uint8_t _reserved2[4]; 615 616 uint64_t mtcap_sensor_map; 617 }; 618 619 struct mcx_reg_mtmp { 620 uint8_t _reserved1[2]; 621 uint16_t mtmp_sensor_index; 622 623 uint8_t _reserved2[2]; 624 uint16_t mtmp_temperature; 625 626 uint16_t mtmp_mte_mtr; 627 #define MCX_REG_MTMP_MTE (1 << 15) 628 #define MCX_REG_MTMP_MTR (1 << 14) 629 uint16_t mtmp_max_temperature; 630 631 uint16_t mtmp_tee; 632 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 633 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 634 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 635 uint16_t mtmp_temperature_threshold_hi; 636 637 uint8_t _reserved3[2]; 638 uint16_t mtmp_temperature_threshold_lo; 639 640 uint8_t _reserved4[4]; 641 642 uint8_t mtmp_sensor_name[8]; 643 }; 644 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 645 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 646 647 #define MCX_MCIA_EEPROM_BYTES 32 648 struct mcx_reg_mcia { 649 uint8_t rm_l; 650 uint8_t rm_module; 651 uint8_t rm_reserved0; 652 uint8_t rm_status; 653 uint8_t rm_i2c_addr; 654 uint8_t rm_page_num; 655 uint16_t rm_dev_addr; 656 uint16_t rm_reserved1; 657 uint16_t rm_size; 658 uint32_t rm_reserved2; 659 uint8_t rm_data[48]; 660 } __packed __aligned(4); 661 662 struct mcx_cmd_query_issi_in { 663 uint16_t cmd_opcode; 664 uint8_t cmd_reserved0[4]; 665 uint16_t cmd_op_mod; 666 uint8_t cmd_reserved1[8]; 667 } __packed __aligned(4); 668 669 struct mcx_cmd_query_issi_il_out { 670 uint8_t cmd_status; 671 uint8_t cmd_reserved0[3]; 672 uint32_t cmd_syndrome; 673 uint8_t cmd_reserved1[2]; 674 uint16_t cmd_current_issi; 675 uint8_t cmd_reserved2[4]; 676 } __packed __aligned(4); 677 678 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 679 680 struct mcx_cmd_query_issi_mb_out { 681 uint8_t cmd_reserved2[16]; 682 uint8_t cmd_supported_issi[80]; /* very big endian */ 683 } __packed __aligned(4); 684 685 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 686 687 struct mcx_cmd_set_issi_in { 688 uint16_t cmd_opcode; 689 uint8_t cmd_reserved0[4]; 690 uint16_t cmd_op_mod; 691 uint8_t cmd_reserved1[2]; 692 uint16_t cmd_current_issi; 693 uint8_t cmd_reserved2[4]; 694 } __packed __aligned(4); 695 696 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 697 698 struct mcx_cmd_set_issi_out { 699 uint8_t cmd_status; 700 uint8_t cmd_reserved0[3]; 701 uint32_t cmd_syndrome; 702 uint8_t cmd_reserved1[8]; 703 } __packed __aligned(4); 704 705 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 706 707 struct mcx_cmd_query_pages_in { 708 uint16_t cmd_opcode; 709 uint8_t cmd_reserved0[4]; 710 uint16_t cmd_op_mod; 711 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 712 #define MCX_CMD_QUERY_PAGES_INIT 0x02 713 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 714 uint8_t cmd_reserved1[8]; 715 } __packed __aligned(4); 716 717 struct mcx_cmd_query_pages_out { 718 uint8_t cmd_status; 719 uint8_t cmd_reserved0[3]; 720 uint32_t cmd_syndrome; 721 uint8_t cmd_reserved1[2]; 722 uint16_t cmd_func_id; 723 int32_t cmd_num_pages; 724 } __packed __aligned(4); 725 726 struct mcx_cmd_manage_pages_in { 727 uint16_t cmd_opcode; 728 uint8_t cmd_reserved0[4]; 729 uint16_t cmd_op_mod; 730 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 731 0x00 732 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 733 0x01 734 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 735 0x02 736 uint8_t cmd_reserved1[2]; 737 uint16_t cmd_func_id; 738 uint32_t cmd_input_num_entries; 739 } __packed __aligned(4); 740 741 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 742 743 struct mcx_cmd_manage_pages_out { 744 uint8_t cmd_status; 745 uint8_t cmd_reserved0[3]; 746 uint32_t cmd_syndrome; 747 uint32_t cmd_output_num_entries; 748 uint8_t cmd_reserved1[4]; 749 } __packed __aligned(4); 750 751 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 752 753 struct mcx_cmd_query_hca_cap_in { 754 uint16_t cmd_opcode; 755 uint8_t cmd_reserved0[4]; 756 uint16_t cmd_op_mod; 757 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 758 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 759 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 760 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 761 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 762 uint8_t cmd_reserved1[8]; 763 } __packed __aligned(4); 764 765 struct mcx_cmd_query_hca_cap_out { 766 uint8_t cmd_status; 767 uint8_t cmd_reserved0[3]; 768 uint32_t cmd_syndrome; 769 uint8_t cmd_reserved1[8]; 770 } __packed __aligned(4); 771 772 #define MCX_HCA_CAP_LEN 0x1000 773 #define MCX_HCA_CAP_NMAILBOXES \ 774 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 775 776 #if __GNUC_PREREQ__(4, 3) 777 #define __counter__ __COUNTER__ 778 #else 779 #define __counter__ __LINE__ 780 #endif 781 782 #define __token(_tok, _num) _tok##_num 783 #define _token(_tok, _num) __token(_tok, _num) 784 #define __reserved__ _token(__reserved, __counter__) 785 786 struct mcx_cap_device { 787 uint8_t reserved0[16]; 788 789 uint8_t log_max_srq_sz; 790 uint8_t log_max_qp_sz; 791 uint8_t __reserved__[1]; 792 uint8_t log_max_qp; /* 5 bits */ 793 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 794 795 uint8_t __reserved__[1]; 796 uint8_t log_max_srq; /* 5 bits */ 797 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 798 uint8_t __reserved__[2]; 799 800 uint8_t __reserved__[1]; 801 uint8_t log_max_cq_sz; 802 uint8_t __reserved__[1]; 803 uint8_t log_max_cq; /* 5 bits */ 804 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 805 806 uint8_t log_max_eq_sz; 807 uint8_t log_max_mkey; /* 6 bits */ 808 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 809 uint8_t __reserved__[1]; 810 uint8_t log_max_eq; /* 4 bits */ 811 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 812 813 uint8_t max_indirection; 814 uint8_t log_max_mrw_sz; /* 7 bits */ 815 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 816 uint8_t teardown_log_max_msf_list_size; 817 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 818 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 819 0x3f 820 uint8_t log_max_klm_list_size; /* 6 bits */ 821 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 822 0x3f 823 824 uint8_t __reserved__[1]; 825 uint8_t log_max_ra_req_dc; /* 6 bits */ 826 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 827 uint8_t __reserved__[1]; 828 uint8_t log_max_ra_res_dc; /* 6 bits */ 829 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 830 0x3f 831 832 uint8_t __reserved__[1]; 833 uint8_t log_max_ra_req_qp; /* 6 bits */ 834 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 835 0x3f 836 uint8_t __reserved__[1]; 837 uint8_t log_max_ra_res_qp; /* 6 bits */ 838 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 839 0x3f 840 841 uint8_t flags1; 842 #define MCX_CAP_DEVICE_END_PAD 0x80 843 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 844 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 845 0x20 846 #define MCX_CAP_DEVICE_START_PAD 0x10 847 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 848 0x08 849 uint8_t __reserved__[1]; 850 uint16_t gid_table_size; 851 852 uint16_t flags2; 853 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 854 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 855 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 856 0x2000 857 #define MCX_CAP_DEVICE_DEBUG 0x1000 858 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 859 0x8000 860 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 861 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 862 uint16_t pkey_table_size; 863 864 uint8_t flags3; 865 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 866 0x80 867 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 868 0x40 869 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 870 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 871 #define MCX_CAP_DEVICE_ETS 0x04 872 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 873 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 874 0x01 875 uint8_t local_ca_ack_delay; /* 5 bits */ 876 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 877 0x1f 878 #define MCX_CAP_DEVICE_MCAM_REG 0x40 879 uint8_t port_type; 880 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 881 0x80 882 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 883 #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01 884 uint8_t num_ports; 885 886 uint8_t snapshot_log_max_msg; 887 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 888 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 889 uint8_t max_tc; /* 4 bits */ 890 #define MCX_CAP_DEVICE_MAX_TC 0x0f 891 uint8_t flags4; 892 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 893 #define MCX_CAP_DEVICE_DCBX 0x40 894 #define MCX_CAP_DEVICE_ROL_S 0x02 895 #define MCX_CAP_DEVICE_ROL_G 0x01 896 uint8_t wol; 897 #define MCX_CAP_DEVICE_WOL_S 0x40 898 #define MCX_CAP_DEVICE_WOL_G 0x20 899 #define MCX_CAP_DEVICE_WOL_A 0x10 900 #define MCX_CAP_DEVICE_WOL_B 0x08 901 #define MCX_CAP_DEVICE_WOL_M 0x04 902 #define MCX_CAP_DEVICE_WOL_U 0x02 903 #define MCX_CAP_DEVICE_WOL_P 0x01 904 905 uint16_t stat_rate_support; 906 uint8_t __reserved__[1]; 907 uint8_t cqe_version; /* 4 bits */ 908 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 909 910 uint32_t flags5; 911 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 912 0x80000000 913 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 914 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 915 0x10000000 916 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 917 0x08000000 918 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 919 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 920 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 921 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 922 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 923 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 924 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 925 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 926 #define MCX_CAP_DEVICE_SHO 0x00000100 927 #define MCX_CAP_DEVICE_TPH 0x00000080 928 #define MCX_CAP_DEVICE_RF 0x00000040 929 #define MCX_CAP_DEVICE_DCT 0x00000020 930 #define MCX_CAP_DEVICE_QOS 0x00000010 931 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 932 #define MCX_CAP_DEVICE_ROCE 0x00000004 933 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 934 935 uint32_t flags6; 936 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 937 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 938 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 939 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 940 0x10000000 941 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 942 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 943 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 944 #define MCX_CAP_DEVICE_PG 0x01000000 945 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 946 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 947 0x00400000 948 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 949 0x00200000 950 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 951 0x00100000 952 #define MCX_CAP_DEVICE_CD 0x00080000 953 #define MCX_CAP_DEVICE_ATM 0x00040000 954 #define MCX_CAP_DEVICE_APM 0x00020000 955 #define MCX_CAP_DEVICE_IMAICL 0x00010000 956 #define MCX_CAP_DEVICE_QKV 0x00000200 957 #define MCX_CAP_DEVICE_PKV 0x00000100 958 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 959 #define MCX_CAP_DEVICE_XRC 0x00000008 960 #define MCX_CAP_DEVICE_UD 0x00000004 961 #define MCX_CAP_DEVICE_UC 0x00000002 962 #define MCX_CAP_DEVICE_RC 0x00000001 963 964 uint8_t uar_flags; 965 #define MCX_CAP_DEVICE_UAR_4K 0x80 966 uint8_t uar_sz; /* 6 bits */ 967 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 968 uint8_t __reserved__[1]; 969 uint8_t log_pg_sz; 970 971 uint8_t flags7; 972 #define MCX_CAP_DEVICE_BF 0x80 973 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 974 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 975 0x20 976 uint8_t log_bf_reg_size; /* 5 bits */ 977 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 978 uint8_t __reserved__[2]; 979 980 uint16_t num_of_diagnostic_counters; 981 uint16_t max_wqe_sz_sq; 982 983 uint8_t __reserved__[2]; 984 uint16_t max_wqe_sz_rq; 985 986 uint8_t __reserved__[2]; 987 uint16_t max_wqe_sz_sq_dc; 988 989 uint32_t max_qp_mcg; /* 25 bits */ 990 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 991 992 uint8_t __reserved__[3]; 993 uint8_t log_max_mcq; 994 995 uint8_t log_max_transport_domain; /* 5 bits */ 996 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 997 0x1f 998 uint8_t log_max_pd; /* 5 bits */ 999 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 1000 uint8_t __reserved__[1]; 1001 uint8_t log_max_xrcd; /* 5 bits */ 1002 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 1003 1004 uint8_t __reserved__[2]; 1005 uint16_t max_flow_counter; 1006 1007 uint8_t log_max_rq; /* 5 bits */ 1008 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 1009 uint8_t log_max_sq; /* 5 bits */ 1010 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 1011 uint8_t log_max_tir; /* 5 bits */ 1012 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 1013 uint8_t log_max_tis; /* 5 bits */ 1014 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 1015 1016 uint8_t flags8; 1017 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 1018 0x80 1019 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 1020 uint8_t log_max_rqt; /* 5 bits */ 1021 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 1022 uint8_t log_max_rqt_size; /* 5 bits */ 1023 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 1024 uint8_t log_max_tis_per_sq; /* 5 bits */ 1025 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 1026 0x1f 1027 1028 uint8_t flags9; 1029 #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \ 1030 0x80 1031 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \ 1032 0x1f 1033 uint8_t log_min_stride_sz_rq; /* 5 bits */ 1034 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \ 1035 0x1f 1036 uint8_t log_max_stride_sz_sq; /* 5 bits */ 1037 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \ 1038 0x1f 1039 uint8_t log_min_stride_sz_sq; /* 5 bits */ 1040 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \ 1041 0x1f 1042 1043 uint8_t log_max_hairpin_queues; 1044 #define MXC_CAP_DEVICE_HAIRPIN 0x80 1045 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \ 1046 0x1f 1047 uint8_t log_min_hairpin_queues; 1048 #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \ 1049 0x1f 1050 uint8_t log_max_hairpin_num_packets; 1051 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \ 1052 0x1f 1053 uint8_t log_max_mq_sz; 1054 #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \ 1055 0x1f 1056 1057 uint8_t log_min_hairpin_wq_data_sz; 1058 #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \ 1059 0x80 1060 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \ 1061 0x40 1062 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \ 1063 0x20 1064 #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \ 1065 0x1f 1066 uint8_t log_max_vlan_list; 1067 #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \ 1068 0x80 1069 #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \ 1070 0x1f 1071 uint8_t log_max_current_mc_list; 1072 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \ 1073 0x1f 1074 uint8_t log_max_current_uc_list; 1075 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \ 1076 0x1f 1077 1078 uint8_t __reserved__[4]; 1079 1080 uint32_t create_qp_start_hint; /* 24 bits */ 1081 1082 uint8_t log_max_uctx; /* 5 bits */ 1083 #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f 1084 uint8_t log_max_umem; /* 5 bits */ 1085 #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f 1086 uint16_t max_num_eqs; 1087 1088 uint8_t log_max_l2_table; /* 5 bits */ 1089 #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f 1090 uint8_t __reserved__[1]; 1091 uint16_t log_uar_page_sz; 1092 1093 uint8_t __reserved__[8]; 1094 1095 uint32_t device_frequency_mhz; 1096 uint32_t device_frequency_khz; 1097 } __packed __aligned(8); 1098 1099 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 1100 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 1101 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 1102 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 1103 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 1104 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 1105 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98); 1106 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c); 1107 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 1108 1109 struct mcx_cmd_set_driver_version_in { 1110 uint16_t cmd_opcode; 1111 uint8_t cmd_reserved0[4]; 1112 uint16_t cmd_op_mod; 1113 uint8_t cmd_reserved1[8]; 1114 } __packed __aligned(4); 1115 1116 struct mcx_cmd_set_driver_version_out { 1117 uint8_t cmd_status; 1118 uint8_t cmd_reserved0[3]; 1119 uint32_t cmd_syndrome; 1120 uint8_t cmd_reserved1[8]; 1121 } __packed __aligned(4); 1122 1123 struct mcx_cmd_set_driver_version { 1124 uint8_t cmd_driver_version[64]; 1125 } __packed __aligned(8); 1126 1127 struct mcx_cmd_modify_nic_vport_context_in { 1128 uint16_t cmd_opcode; 1129 uint8_t cmd_reserved0[4]; 1130 uint16_t cmd_op_mod; 1131 uint8_t cmd_reserved1[4]; 1132 uint32_t cmd_field_select; 1133 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1134 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1135 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1136 } __packed __aligned(4); 1137 1138 struct mcx_cmd_modify_nic_vport_context_out { 1139 uint8_t cmd_status; 1140 uint8_t cmd_reserved0[3]; 1141 uint32_t cmd_syndrome; 1142 uint8_t cmd_reserved1[8]; 1143 } __packed __aligned(4); 1144 1145 struct mcx_cmd_query_nic_vport_context_in { 1146 uint16_t cmd_opcode; 1147 uint8_t cmd_reserved0[4]; 1148 uint16_t cmd_op_mod; 1149 uint8_t cmd_reserved1[4]; 1150 uint8_t cmd_allowed_list_type; 1151 uint8_t cmd_reserved2[3]; 1152 } __packed __aligned(4); 1153 1154 struct mcx_cmd_query_nic_vport_context_out { 1155 uint8_t cmd_status; 1156 uint8_t cmd_reserved0[3]; 1157 uint32_t cmd_syndrome; 1158 uint8_t cmd_reserved1[8]; 1159 } __packed __aligned(4); 1160 1161 struct mcx_nic_vport_ctx { 1162 uint32_t vp_min_wqe_inline_mode; 1163 uint8_t vp_reserved0[32]; 1164 uint32_t vp_mtu; 1165 uint8_t vp_reserved1[200]; 1166 uint16_t vp_flags; 1167 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1168 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1169 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1170 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1171 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1172 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1173 uint16_t vp_allowed_list_size; 1174 uint64_t vp_perm_addr; 1175 uint8_t vp_reserved2[4]; 1176 /* allowed list follows */ 1177 } __packed __aligned(4); 1178 1179 struct mcx_counter { 1180 uint64_t packets; 1181 uint64_t octets; 1182 } __packed __aligned(4); 1183 1184 struct mcx_nic_vport_counters { 1185 struct mcx_counter rx_err; 1186 struct mcx_counter tx_err; 1187 uint8_t reserved0[64]; /* 0x30 */ 1188 struct mcx_counter rx_bcast; 1189 struct mcx_counter tx_bcast; 1190 struct mcx_counter rx_ucast; 1191 struct mcx_counter tx_ucast; 1192 struct mcx_counter rx_mcast; 1193 struct mcx_counter tx_mcast; 1194 uint8_t reserved1[0x210 - 0xd0]; 1195 } __packed __aligned(4); 1196 1197 struct mcx_cmd_query_vport_counters_in { 1198 uint16_t cmd_opcode; 1199 uint8_t cmd_reserved0[4]; 1200 uint16_t cmd_op_mod; 1201 uint8_t cmd_reserved1[8]; 1202 } __packed __aligned(4); 1203 1204 struct mcx_cmd_query_vport_counters_mb_in { 1205 uint8_t cmd_reserved0[8]; 1206 uint8_t cmd_clear; 1207 uint8_t cmd_reserved1[7]; 1208 } __packed __aligned(4); 1209 1210 struct mcx_cmd_query_vport_counters_out { 1211 uint8_t cmd_status; 1212 uint8_t cmd_reserved0[3]; 1213 uint32_t cmd_syndrome; 1214 uint8_t cmd_reserved1[8]; 1215 } __packed __aligned(4); 1216 1217 struct mcx_cmd_query_flow_counter_in { 1218 uint16_t cmd_opcode; 1219 uint8_t cmd_reserved0[4]; 1220 uint16_t cmd_op_mod; 1221 uint8_t cmd_reserved1[8]; 1222 } __packed __aligned(4); 1223 1224 struct mcx_cmd_query_flow_counter_mb_in { 1225 uint8_t cmd_reserved0[8]; 1226 uint8_t cmd_clear; 1227 uint8_t cmd_reserved1[5]; 1228 uint16_t cmd_flow_counter_id; 1229 } __packed __aligned(4); 1230 1231 struct mcx_cmd_query_flow_counter_out { 1232 uint8_t cmd_status; 1233 uint8_t cmd_reserved0[3]; 1234 uint32_t cmd_syndrome; 1235 uint8_t cmd_reserved1[8]; 1236 } __packed __aligned(4); 1237 1238 struct mcx_cmd_alloc_uar_in { 1239 uint16_t cmd_opcode; 1240 uint8_t cmd_reserved0[4]; 1241 uint16_t cmd_op_mod; 1242 uint8_t cmd_reserved1[8]; 1243 } __packed __aligned(4); 1244 1245 struct mcx_cmd_alloc_uar_out { 1246 uint8_t cmd_status; 1247 uint8_t cmd_reserved0[3]; 1248 uint32_t cmd_syndrome; 1249 uint32_t cmd_uar; 1250 uint8_t cmd_reserved1[4]; 1251 } __packed __aligned(4); 1252 1253 struct mcx_cmd_query_special_ctx_in { 1254 uint16_t cmd_opcode; 1255 uint8_t cmd_reserved0[4]; 1256 uint16_t cmd_op_mod; 1257 uint8_t cmd_reserved1[8]; 1258 } __packed __aligned(4); 1259 1260 struct mcx_cmd_query_special_ctx_out { 1261 uint8_t cmd_status; 1262 uint8_t cmd_reserved0[3]; 1263 uint32_t cmd_syndrome; 1264 uint8_t cmd_reserved1[4]; 1265 uint32_t cmd_resd_lkey; 1266 } __packed __aligned(4); 1267 1268 struct mcx_eq_ctx { 1269 uint32_t eq_status; 1270 #define MCX_EQ_CTX_STATE_SHIFT 8 1271 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT) 1272 #define MCX_EQ_CTX_STATE_ARMED 0x9 1273 #define MCX_EQ_CTX_STATE_FIRED 0xa 1274 #define MCX_EQ_CTX_OI_SHIFT 17 1275 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1276 #define MCX_EQ_CTX_EC_SHIFT 18 1277 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1278 #define MCX_EQ_CTX_STATUS_SHIFT 28 1279 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1280 #define MCX_EQ_CTX_STATUS_OK 0x0 1281 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa 1282 uint32_t eq_reserved1; 1283 uint32_t eq_page_offset; 1284 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1285 uint32_t eq_uar_size; 1286 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1287 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1288 uint32_t eq_reserved2; 1289 uint8_t eq_reserved3[3]; 1290 uint8_t eq_intr; 1291 uint32_t eq_log_page_size; 1292 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1293 uint32_t eq_reserved4[3]; 1294 uint32_t eq_consumer_counter; 1295 uint32_t eq_producer_counter; 1296 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1297 uint32_t eq_reserved5[4]; 1298 } __packed __aligned(4); 1299 1300 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1301 1302 struct mcx_cmd_create_eq_in { 1303 uint16_t cmd_opcode; 1304 uint8_t cmd_reserved0[4]; 1305 uint16_t cmd_op_mod; 1306 uint8_t cmd_reserved1[8]; 1307 } __packed __aligned(4); 1308 1309 struct mcx_cmd_create_eq_mb_in { 1310 struct mcx_eq_ctx cmd_eq_ctx; 1311 uint8_t cmd_reserved0[8]; 1312 uint64_t cmd_event_bitmask; 1313 #define MCX_EVENT_TYPE_COMPLETION 0x00 1314 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1315 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1316 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1317 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1318 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1319 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1320 uint8_t cmd_reserved1[176]; 1321 } __packed __aligned(4); 1322 1323 struct mcx_cmd_create_eq_out { 1324 uint8_t cmd_status; 1325 uint8_t cmd_reserved0[3]; 1326 uint32_t cmd_syndrome; 1327 uint32_t cmd_eqn; 1328 uint8_t cmd_reserved1[4]; 1329 } __packed __aligned(4); 1330 1331 struct mcx_cmd_query_eq_in { 1332 uint16_t cmd_opcode; 1333 uint8_t cmd_reserved0[4]; 1334 uint16_t cmd_op_mod; 1335 uint32_t cmd_eqn; 1336 uint8_t cmd_reserved1[4]; 1337 } __packed __aligned(4); 1338 1339 struct mcx_cmd_query_eq_out { 1340 uint8_t cmd_status; 1341 uint8_t cmd_reserved0[3]; 1342 uint32_t cmd_syndrome; 1343 uint8_t cmd_reserved1[8]; 1344 } __packed __aligned(4); 1345 1346 struct mcx_eq_entry { 1347 uint8_t eq_reserved1; 1348 uint8_t eq_event_type; 1349 uint8_t eq_reserved2; 1350 uint8_t eq_event_sub_type; 1351 1352 uint8_t eq_reserved3[28]; 1353 uint32_t eq_event_data[7]; 1354 uint8_t eq_reserved4[2]; 1355 uint8_t eq_signature; 1356 uint8_t eq_owner; 1357 #define MCX_EQ_ENTRY_OWNER_INIT 1 1358 } __packed __aligned(4); 1359 1360 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1361 1362 struct mcx_cmd_alloc_pd_in { 1363 uint16_t cmd_opcode; 1364 uint8_t cmd_reserved0[4]; 1365 uint16_t cmd_op_mod; 1366 uint8_t cmd_reserved1[8]; 1367 } __packed __aligned(4); 1368 1369 struct mcx_cmd_alloc_pd_out { 1370 uint8_t cmd_status; 1371 uint8_t cmd_reserved0[3]; 1372 uint32_t cmd_syndrome; 1373 uint32_t cmd_pd; 1374 uint8_t cmd_reserved1[4]; 1375 } __packed __aligned(4); 1376 1377 struct mcx_cmd_alloc_td_in { 1378 uint16_t cmd_opcode; 1379 uint8_t cmd_reserved0[4]; 1380 uint16_t cmd_op_mod; 1381 uint8_t cmd_reserved1[8]; 1382 } __packed __aligned(4); 1383 1384 struct mcx_cmd_alloc_td_out { 1385 uint8_t cmd_status; 1386 uint8_t cmd_reserved0[3]; 1387 uint32_t cmd_syndrome; 1388 uint32_t cmd_tdomain; 1389 uint8_t cmd_reserved1[4]; 1390 } __packed __aligned(4); 1391 1392 struct mcx_cmd_create_tir_in { 1393 uint16_t cmd_opcode; 1394 uint8_t cmd_reserved0[4]; 1395 uint16_t cmd_op_mod; 1396 uint8_t cmd_reserved1[8]; 1397 } __packed __aligned(4); 1398 1399 struct mcx_cmd_create_tir_mb_in { 1400 uint8_t cmd_reserved0[20]; 1401 uint32_t cmd_disp_type; 1402 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1403 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1404 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1405 uint8_t cmd_reserved1[8]; 1406 uint32_t cmd_lro; 1407 uint8_t cmd_reserved2[8]; 1408 uint32_t cmd_inline_rqn; 1409 uint32_t cmd_indir_table; 1410 uint32_t cmd_tdomain; 1411 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1412 #define MCX_TIR_CTX_HASH_SHIFT 28 1413 uint8_t cmd_rx_hash_key[40]; 1414 uint32_t cmd_rx_hash_sel_outer; 1415 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1416 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1417 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1418 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1419 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1420 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1421 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1422 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1423 uint32_t cmd_rx_hash_sel_inner; 1424 uint8_t cmd_reserved3[152]; 1425 } __packed __aligned(4); 1426 1427 struct mcx_cmd_create_tir_out { 1428 uint8_t cmd_status; 1429 uint8_t cmd_reserved0[3]; 1430 uint32_t cmd_syndrome; 1431 uint32_t cmd_tirn; 1432 uint8_t cmd_reserved1[4]; 1433 } __packed __aligned(4); 1434 1435 struct mcx_cmd_destroy_tir_in { 1436 uint16_t cmd_opcode; 1437 uint8_t cmd_reserved0[4]; 1438 uint16_t cmd_op_mod; 1439 uint32_t cmd_tirn; 1440 uint8_t cmd_reserved1[4]; 1441 } __packed __aligned(4); 1442 1443 struct mcx_cmd_destroy_tir_out { 1444 uint8_t cmd_status; 1445 uint8_t cmd_reserved0[3]; 1446 uint32_t cmd_syndrome; 1447 uint8_t cmd_reserved1[8]; 1448 } __packed __aligned(4); 1449 1450 struct mcx_cmd_create_tis_in { 1451 uint16_t cmd_opcode; 1452 uint8_t cmd_reserved0[4]; 1453 uint16_t cmd_op_mod; 1454 uint8_t cmd_reserved1[8]; 1455 } __packed __aligned(4); 1456 1457 struct mcx_cmd_create_tis_mb_in { 1458 uint8_t cmd_reserved[16]; 1459 uint32_t cmd_prio; 1460 uint8_t cmd_reserved1[32]; 1461 uint32_t cmd_tdomain; 1462 uint8_t cmd_reserved2[120]; 1463 } __packed __aligned(4); 1464 1465 struct mcx_cmd_create_tis_out { 1466 uint8_t cmd_status; 1467 uint8_t cmd_reserved0[3]; 1468 uint32_t cmd_syndrome; 1469 uint32_t cmd_tisn; 1470 uint8_t cmd_reserved1[4]; 1471 } __packed __aligned(4); 1472 1473 struct mcx_cmd_destroy_tis_in { 1474 uint16_t cmd_opcode; 1475 uint8_t cmd_reserved0[4]; 1476 uint16_t cmd_op_mod; 1477 uint32_t cmd_tisn; 1478 uint8_t cmd_reserved1[4]; 1479 } __packed __aligned(4); 1480 1481 struct mcx_cmd_destroy_tis_out { 1482 uint8_t cmd_status; 1483 uint8_t cmd_reserved0[3]; 1484 uint32_t cmd_syndrome; 1485 uint8_t cmd_reserved1[8]; 1486 } __packed __aligned(4); 1487 1488 struct mcx_cmd_create_rqt_in { 1489 uint16_t cmd_opcode; 1490 uint8_t cmd_reserved0[4]; 1491 uint16_t cmd_op_mod; 1492 uint8_t cmd_reserved1[8]; 1493 } __packed __aligned(4); 1494 1495 struct mcx_rqt_ctx { 1496 uint8_t cmd_reserved0[20]; 1497 uint16_t cmd_reserved1; 1498 uint16_t cmd_rqt_max_size; 1499 uint16_t cmd_reserved2; 1500 uint16_t cmd_rqt_actual_size; 1501 uint8_t cmd_reserved3[212]; 1502 } __packed __aligned(4); 1503 1504 struct mcx_cmd_create_rqt_mb_in { 1505 uint8_t cmd_reserved0[16]; 1506 struct mcx_rqt_ctx cmd_rqt; 1507 } __packed __aligned(4); 1508 1509 struct mcx_cmd_create_rqt_out { 1510 uint8_t cmd_status; 1511 uint8_t cmd_reserved0[3]; 1512 uint32_t cmd_syndrome; 1513 uint32_t cmd_rqtn; 1514 uint8_t cmd_reserved1[4]; 1515 } __packed __aligned(4); 1516 1517 struct mcx_cmd_destroy_rqt_in { 1518 uint16_t cmd_opcode; 1519 uint8_t cmd_reserved0[4]; 1520 uint16_t cmd_op_mod; 1521 uint32_t cmd_rqtn; 1522 uint8_t cmd_reserved1[4]; 1523 } __packed __aligned(4); 1524 1525 struct mcx_cmd_destroy_rqt_out { 1526 uint8_t cmd_status; 1527 uint8_t cmd_reserved0[3]; 1528 uint32_t cmd_syndrome; 1529 uint8_t cmd_reserved1[8]; 1530 } __packed __aligned(4); 1531 1532 struct mcx_cq_ctx { 1533 uint32_t cq_status; 1534 #define MCX_CQ_CTX_STATUS_SHIFT 28 1535 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT) 1536 #define MCX_CQ_CTX_STATUS_OK 0x0 1537 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9 1538 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa 1539 #define MCX_CQ_CTX_STATE_SHIFT 8 1540 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT) 1541 #define MCX_CQ_CTX_STATE_SOLICITED 0x6 1542 #define MCX_CQ_CTX_STATE_ARMED 0x9 1543 #define MCX_CQ_CTX_STATE_FIRED 0xa 1544 uint32_t cq_reserved1; 1545 uint32_t cq_page_offset; 1546 uint32_t cq_uar_size; 1547 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1548 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1549 uint32_t cq_period_max_count; 1550 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1551 uint32_t cq_eqn; 1552 uint32_t cq_log_page_size; 1553 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1554 uint32_t cq_reserved2; 1555 uint32_t cq_last_notified; 1556 uint32_t cq_last_solicit; 1557 uint32_t cq_consumer_counter; 1558 uint32_t cq_producer_counter; 1559 uint8_t cq_reserved3[8]; 1560 uint64_t cq_doorbell; 1561 } __packed __aligned(4); 1562 1563 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1564 1565 struct mcx_cmd_create_cq_in { 1566 uint16_t cmd_opcode; 1567 uint8_t cmd_reserved0[4]; 1568 uint16_t cmd_op_mod; 1569 uint8_t cmd_reserved1[8]; 1570 } __packed __aligned(4); 1571 1572 struct mcx_cmd_create_cq_mb_in { 1573 struct mcx_cq_ctx cmd_cq_ctx; 1574 uint8_t cmd_reserved1[192]; 1575 } __packed __aligned(4); 1576 1577 struct mcx_cmd_create_cq_out { 1578 uint8_t cmd_status; 1579 uint8_t cmd_reserved0[3]; 1580 uint32_t cmd_syndrome; 1581 uint32_t cmd_cqn; 1582 uint8_t cmd_reserved1[4]; 1583 } __packed __aligned(4); 1584 1585 struct mcx_cmd_destroy_cq_in { 1586 uint16_t cmd_opcode; 1587 uint8_t cmd_reserved0[4]; 1588 uint16_t cmd_op_mod; 1589 uint32_t cmd_cqn; 1590 uint8_t cmd_reserved1[4]; 1591 } __packed __aligned(4); 1592 1593 struct mcx_cmd_destroy_cq_out { 1594 uint8_t cmd_status; 1595 uint8_t cmd_reserved0[3]; 1596 uint32_t cmd_syndrome; 1597 uint8_t cmd_reserved1[8]; 1598 } __packed __aligned(4); 1599 1600 struct mcx_cmd_query_cq_in { 1601 uint16_t cmd_opcode; 1602 uint8_t cmd_reserved0[4]; 1603 uint16_t cmd_op_mod; 1604 uint32_t cmd_cqn; 1605 uint8_t cmd_reserved1[4]; 1606 } __packed __aligned(4); 1607 1608 struct mcx_cmd_query_cq_out { 1609 uint8_t cmd_status; 1610 uint8_t cmd_reserved0[3]; 1611 uint32_t cmd_syndrome; 1612 uint8_t cmd_reserved1[8]; 1613 } __packed __aligned(4); 1614 1615 struct mcx_cq_entry { 1616 uint32_t __reserved__; 1617 uint32_t cq_lro; 1618 uint32_t cq_lro_ack_seq_num; 1619 uint32_t cq_rx_hash; 1620 uint8_t cq_rx_hash_type; 1621 uint8_t cq_ml_path; 1622 uint16_t __reserved__; 1623 uint32_t cq_checksum; 1624 uint32_t __reserved__; 1625 uint32_t cq_flags; 1626 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1627 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1628 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1629 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1630 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1631 1632 uint32_t cq_lro_srqn; 1633 uint32_t __reserved__[2]; 1634 uint32_t cq_byte_cnt; 1635 uint64_t cq_timestamp; 1636 uint8_t cq_rx_drops; 1637 uint8_t cq_flow_tag[3]; 1638 uint16_t cq_wqe_count; 1639 uint8_t cq_signature; 1640 uint8_t cq_opcode_owner; 1641 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1642 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1643 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1644 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1645 1646 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1647 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1648 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1649 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1650 1651 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1652 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1653 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1654 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1655 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1656 1657 } __packed __aligned(4); 1658 1659 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1660 1661 struct mcx_cq_doorbell { 1662 uint32_t db_update_ci; 1663 uint32_t db_arm_ci; 1664 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1665 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1666 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1667 } __packed __aligned(8); 1668 1669 struct mcx_wq_ctx { 1670 uint8_t wq_type; 1671 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1672 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1673 uint8_t wq_reserved0[5]; 1674 uint16_t wq_lwm; 1675 uint32_t wq_pd; 1676 uint32_t wq_uar_page; 1677 uint64_t wq_doorbell; 1678 uint32_t wq_hw_counter; 1679 uint32_t wq_sw_counter; 1680 uint16_t wq_log_stride; 1681 uint8_t wq_log_page_sz; 1682 uint8_t wq_log_size; 1683 uint8_t wq_reserved1[156]; 1684 } __packed __aligned(4); 1685 1686 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1687 1688 struct mcx_sq_ctx { 1689 uint32_t sq_flags; 1690 #define MCX_SQ_CTX_RLKEY (1U << 31) 1691 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1692 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1693 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1694 #define MCX_SQ_CTX_STATE_SHIFT 20 1695 #define MCX_SQ_CTX_STATE_MASK (0xf << 20) 1696 #define MCX_SQ_CTX_STATE_RST 0 1697 #define MCX_SQ_CTX_STATE_RDY 1 1698 #define MCX_SQ_CTX_STATE_ERR 3 1699 uint32_t sq_user_index; 1700 uint32_t sq_cqn; 1701 uint32_t sq_reserved1[5]; 1702 uint32_t sq_tis_lst_sz; 1703 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1704 uint32_t sq_reserved2[2]; 1705 uint32_t sq_tis_num; 1706 struct mcx_wq_ctx sq_wq; 1707 } __packed __aligned(4); 1708 1709 struct mcx_sq_entry_seg { 1710 uint32_t sqs_byte_count; 1711 uint32_t sqs_lkey; 1712 uint64_t sqs_addr; 1713 } __packed __aligned(4); 1714 1715 struct mcx_sq_entry { 1716 /* control segment */ 1717 uint32_t sqe_opcode_index; 1718 #define MCX_SQE_WQE_INDEX_SHIFT 8 1719 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1720 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1721 uint32_t sqe_ds_sq_num; 1722 #define MCX_SQE_SQ_NUM_SHIFT 8 1723 uint32_t sqe_signature; 1724 #define MCX_SQE_SIGNATURE_SHIFT 24 1725 #define MCX_SQE_SOLICITED_EVENT 0x02 1726 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1727 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1728 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1729 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1730 #define MCX_SQE_FM_NO_FENCE 0x00 1731 #define MCX_SQE_FM_SMALL_FENCE 0x40 1732 uint32_t sqe_mkey; 1733 1734 /* ethernet segment */ 1735 uint32_t sqe_reserved1; 1736 uint32_t sqe_mss_csum; 1737 #define MCX_SQE_L4_CSUM (1 << 31) 1738 #define MCX_SQE_L3_CSUM (1 << 30) 1739 uint32_t sqe_reserved2; 1740 uint16_t sqe_inline_header_size; 1741 uint16_t sqe_inline_headers[9]; 1742 1743 /* data segment */ 1744 struct mcx_sq_entry_seg sqe_segs[1]; 1745 } __packed __aligned(64); 1746 1747 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1748 1749 struct mcx_cmd_create_sq_in { 1750 uint16_t cmd_opcode; 1751 uint8_t cmd_reserved0[4]; 1752 uint16_t cmd_op_mod; 1753 uint8_t cmd_reserved1[8]; 1754 } __packed __aligned(4); 1755 1756 struct mcx_cmd_create_sq_out { 1757 uint8_t cmd_status; 1758 uint8_t cmd_reserved0[3]; 1759 uint32_t cmd_syndrome; 1760 uint32_t cmd_sqn; 1761 uint8_t cmd_reserved1[4]; 1762 } __packed __aligned(4); 1763 1764 struct mcx_cmd_modify_sq_in { 1765 uint16_t cmd_opcode; 1766 uint8_t cmd_reserved0[4]; 1767 uint16_t cmd_op_mod; 1768 uint32_t cmd_sq_state; 1769 uint8_t cmd_reserved1[4]; 1770 } __packed __aligned(4); 1771 1772 struct mcx_cmd_modify_sq_mb_in { 1773 uint32_t cmd_modify_hi; 1774 uint32_t cmd_modify_lo; 1775 uint8_t cmd_reserved0[8]; 1776 struct mcx_sq_ctx cmd_sq_ctx; 1777 } __packed __aligned(4); 1778 1779 struct mcx_cmd_modify_sq_out { 1780 uint8_t cmd_status; 1781 uint8_t cmd_reserved0[3]; 1782 uint32_t cmd_syndrome; 1783 uint8_t cmd_reserved1[8]; 1784 } __packed __aligned(4); 1785 1786 struct mcx_cmd_destroy_sq_in { 1787 uint16_t cmd_opcode; 1788 uint8_t cmd_reserved0[4]; 1789 uint16_t cmd_op_mod; 1790 uint32_t cmd_sqn; 1791 uint8_t cmd_reserved1[4]; 1792 } __packed __aligned(4); 1793 1794 struct mcx_cmd_destroy_sq_out { 1795 uint8_t cmd_status; 1796 uint8_t cmd_reserved0[3]; 1797 uint32_t cmd_syndrome; 1798 uint8_t cmd_reserved1[8]; 1799 } __packed __aligned(4); 1800 1801 1802 struct mcx_rq_ctx { 1803 uint32_t rq_flags; 1804 #define MCX_RQ_CTX_RLKEY (1U << 31) 1805 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1806 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1807 #define MCX_RQ_CTX_STATE_SHIFT 20 1808 #define MCX_RQ_CTX_STATE_MASK (0xf << 20) 1809 #define MCX_RQ_CTX_STATE_RST 0 1810 #define MCX_RQ_CTX_STATE_RDY 1 1811 #define MCX_RQ_CTX_STATE_ERR 3 1812 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1813 uint32_t rq_user_index; 1814 uint32_t rq_cqn; 1815 uint32_t rq_reserved1; 1816 uint32_t rq_rmpn; 1817 uint32_t rq_reserved2[7]; 1818 struct mcx_wq_ctx rq_wq; 1819 } __packed __aligned(4); 1820 1821 struct mcx_rq_entry { 1822 uint32_t rqe_byte_count; 1823 uint32_t rqe_lkey; 1824 uint64_t rqe_addr; 1825 } __packed __aligned(16); 1826 1827 struct mcx_cmd_create_rq_in { 1828 uint16_t cmd_opcode; 1829 uint8_t cmd_reserved0[4]; 1830 uint16_t cmd_op_mod; 1831 uint8_t cmd_reserved1[8]; 1832 } __packed __aligned(4); 1833 1834 struct mcx_cmd_create_rq_out { 1835 uint8_t cmd_status; 1836 uint8_t cmd_reserved0[3]; 1837 uint32_t cmd_syndrome; 1838 uint32_t cmd_rqn; 1839 uint8_t cmd_reserved1[4]; 1840 } __packed __aligned(4); 1841 1842 struct mcx_cmd_modify_rq_in { 1843 uint16_t cmd_opcode; 1844 uint8_t cmd_reserved0[4]; 1845 uint16_t cmd_op_mod; 1846 uint32_t cmd_rq_state; 1847 uint8_t cmd_reserved1[4]; 1848 } __packed __aligned(4); 1849 1850 struct mcx_cmd_modify_rq_mb_in { 1851 uint32_t cmd_modify_hi; 1852 uint32_t cmd_modify_lo; 1853 uint8_t cmd_reserved0[8]; 1854 struct mcx_rq_ctx cmd_rq_ctx; 1855 } __packed __aligned(4); 1856 1857 struct mcx_cmd_modify_rq_out { 1858 uint8_t cmd_status; 1859 uint8_t cmd_reserved0[3]; 1860 uint32_t cmd_syndrome; 1861 uint8_t cmd_reserved1[8]; 1862 } __packed __aligned(4); 1863 1864 struct mcx_cmd_destroy_rq_in { 1865 uint16_t cmd_opcode; 1866 uint8_t cmd_reserved0[4]; 1867 uint16_t cmd_op_mod; 1868 uint32_t cmd_rqn; 1869 uint8_t cmd_reserved1[4]; 1870 } __packed __aligned(4); 1871 1872 struct mcx_cmd_destroy_rq_out { 1873 uint8_t cmd_status; 1874 uint8_t cmd_reserved0[3]; 1875 uint32_t cmd_syndrome; 1876 uint8_t cmd_reserved1[8]; 1877 } __packed __aligned(4); 1878 1879 struct mcx_cmd_create_flow_table_in { 1880 uint16_t cmd_opcode; 1881 uint8_t cmd_reserved0[4]; 1882 uint16_t cmd_op_mod; 1883 uint8_t cmd_reserved1[8]; 1884 } __packed __aligned(4); 1885 1886 struct mcx_flow_table_ctx { 1887 uint8_t ft_miss_action; 1888 uint8_t ft_level; 1889 uint8_t ft_reserved0; 1890 uint8_t ft_log_size; 1891 uint32_t ft_table_miss_id; 1892 uint8_t ft_reserved1[28]; 1893 } __packed __aligned(4); 1894 1895 struct mcx_cmd_create_flow_table_mb_in { 1896 uint8_t cmd_table_type; 1897 uint8_t cmd_reserved0[7]; 1898 struct mcx_flow_table_ctx cmd_ctx; 1899 } __packed __aligned(4); 1900 1901 struct mcx_cmd_create_flow_table_out { 1902 uint8_t cmd_status; 1903 uint8_t cmd_reserved0[3]; 1904 uint32_t cmd_syndrome; 1905 uint32_t cmd_table_id; 1906 uint8_t cmd_reserved1[4]; 1907 } __packed __aligned(4); 1908 1909 struct mcx_cmd_destroy_flow_table_in { 1910 uint16_t cmd_opcode; 1911 uint8_t cmd_reserved0[4]; 1912 uint16_t cmd_op_mod; 1913 uint8_t cmd_reserved1[8]; 1914 } __packed __aligned(4); 1915 1916 struct mcx_cmd_destroy_flow_table_mb_in { 1917 uint8_t cmd_table_type; 1918 uint8_t cmd_reserved0[3]; 1919 uint32_t cmd_table_id; 1920 uint8_t cmd_reserved1[40]; 1921 } __packed __aligned(4); 1922 1923 struct mcx_cmd_destroy_flow_table_out { 1924 uint8_t cmd_status; 1925 uint8_t cmd_reserved0[3]; 1926 uint32_t cmd_syndrome; 1927 uint8_t cmd_reserved1[8]; 1928 } __packed __aligned(4); 1929 1930 struct mcx_cmd_set_flow_table_root_in { 1931 uint16_t cmd_opcode; 1932 uint8_t cmd_reserved0[4]; 1933 uint16_t cmd_op_mod; 1934 uint8_t cmd_reserved1[8]; 1935 } __packed __aligned(4); 1936 1937 struct mcx_cmd_set_flow_table_root_mb_in { 1938 uint8_t cmd_table_type; 1939 uint8_t cmd_reserved0[3]; 1940 uint32_t cmd_table_id; 1941 uint8_t cmd_reserved1[56]; 1942 } __packed __aligned(4); 1943 1944 struct mcx_cmd_set_flow_table_root_out { 1945 uint8_t cmd_status; 1946 uint8_t cmd_reserved0[3]; 1947 uint32_t cmd_syndrome; 1948 uint8_t cmd_reserved1[8]; 1949 } __packed __aligned(4); 1950 1951 struct mcx_flow_match { 1952 /* outer headers */ 1953 uint8_t mc_src_mac[6]; 1954 uint16_t mc_ethertype; 1955 uint8_t mc_dest_mac[6]; 1956 uint16_t mc_first_vlan; 1957 uint8_t mc_ip_proto; 1958 uint8_t mc_ip_dscp_ecn; 1959 uint8_t mc_vlan_flags; 1960 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1961 uint8_t mc_tcp_flags; 1962 uint16_t mc_tcp_sport; 1963 uint16_t mc_tcp_dport; 1964 uint32_t mc_reserved0; 1965 uint16_t mc_udp_sport; 1966 uint16_t mc_udp_dport; 1967 uint8_t mc_src_ip[16]; 1968 uint8_t mc_dest_ip[16]; 1969 1970 /* misc parameters */ 1971 uint8_t mc_reserved1[8]; 1972 uint16_t mc_second_vlan; 1973 uint8_t mc_reserved2[2]; 1974 uint8_t mc_second_vlan_flags; 1975 uint8_t mc_reserved3[15]; 1976 uint32_t mc_outer_ipv6_flow_label; 1977 uint8_t mc_reserved4[32]; 1978 1979 uint8_t mc_reserved[384]; 1980 } __packed __aligned(4); 1981 1982 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1983 1984 struct mcx_cmd_create_flow_group_in { 1985 uint16_t cmd_opcode; 1986 uint8_t cmd_reserved0[4]; 1987 uint16_t cmd_op_mod; 1988 uint8_t cmd_reserved1[8]; 1989 } __packed __aligned(4); 1990 1991 struct mcx_cmd_create_flow_group_mb_in { 1992 uint8_t cmd_table_type; 1993 uint8_t cmd_reserved0[3]; 1994 uint32_t cmd_table_id; 1995 uint8_t cmd_reserved1[4]; 1996 uint32_t cmd_start_flow_index; 1997 uint8_t cmd_reserved2[4]; 1998 uint32_t cmd_end_flow_index; 1999 uint8_t cmd_reserved3[23]; 2000 uint8_t cmd_match_criteria_enable; 2001 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 2002 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 2003 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 2004 struct mcx_flow_match cmd_match_criteria; 2005 uint8_t cmd_reserved4[448]; 2006 } __packed __aligned(4); 2007 2008 struct mcx_cmd_create_flow_group_out { 2009 uint8_t cmd_status; 2010 uint8_t cmd_reserved0[3]; 2011 uint32_t cmd_syndrome; 2012 uint32_t cmd_group_id; 2013 uint8_t cmd_reserved1[4]; 2014 } __packed __aligned(4); 2015 2016 struct mcx_flow_ctx { 2017 uint8_t fc_reserved0[4]; 2018 uint32_t fc_group_id; 2019 uint32_t fc_flow_tag; 2020 uint32_t fc_action; 2021 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 2022 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 2023 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 2024 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 2025 uint32_t fc_dest_list_size; 2026 uint32_t fc_counter_list_size; 2027 uint8_t fc_reserved1[40]; 2028 struct mcx_flow_match fc_match_value; 2029 uint8_t fc_reserved2[192]; 2030 } __packed __aligned(4); 2031 2032 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 2033 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 2034 2035 struct mcx_cmd_destroy_flow_group_in { 2036 uint16_t cmd_opcode; 2037 uint8_t cmd_reserved0[4]; 2038 uint16_t cmd_op_mod; 2039 uint8_t cmd_reserved1[8]; 2040 } __packed __aligned(4); 2041 2042 struct mcx_cmd_destroy_flow_group_mb_in { 2043 uint8_t cmd_table_type; 2044 uint8_t cmd_reserved0[3]; 2045 uint32_t cmd_table_id; 2046 uint32_t cmd_group_id; 2047 uint8_t cmd_reserved1[36]; 2048 } __packed __aligned(4); 2049 2050 struct mcx_cmd_destroy_flow_group_out { 2051 uint8_t cmd_status; 2052 uint8_t cmd_reserved0[3]; 2053 uint32_t cmd_syndrome; 2054 uint8_t cmd_reserved1[8]; 2055 } __packed __aligned(4); 2056 2057 struct mcx_cmd_set_flow_table_entry_in { 2058 uint16_t cmd_opcode; 2059 uint8_t cmd_reserved0[4]; 2060 uint16_t cmd_op_mod; 2061 uint8_t cmd_reserved1[8]; 2062 } __packed __aligned(4); 2063 2064 struct mcx_cmd_set_flow_table_entry_mb_in { 2065 uint8_t cmd_table_type; 2066 uint8_t cmd_reserved0[3]; 2067 uint32_t cmd_table_id; 2068 uint32_t cmd_modify_enable_mask; 2069 uint8_t cmd_reserved1[4]; 2070 uint32_t cmd_flow_index; 2071 uint8_t cmd_reserved2[28]; 2072 struct mcx_flow_ctx cmd_flow_ctx; 2073 } __packed __aligned(4); 2074 2075 struct mcx_cmd_set_flow_table_entry_out { 2076 uint8_t cmd_status; 2077 uint8_t cmd_reserved0[3]; 2078 uint32_t cmd_syndrome; 2079 uint8_t cmd_reserved1[8]; 2080 } __packed __aligned(4); 2081 2082 struct mcx_cmd_query_flow_table_entry_in { 2083 uint16_t cmd_opcode; 2084 uint8_t cmd_reserved0[4]; 2085 uint16_t cmd_op_mod; 2086 uint8_t cmd_reserved1[8]; 2087 } __packed __aligned(4); 2088 2089 struct mcx_cmd_query_flow_table_entry_mb_in { 2090 uint8_t cmd_table_type; 2091 uint8_t cmd_reserved0[3]; 2092 uint32_t cmd_table_id; 2093 uint8_t cmd_reserved1[8]; 2094 uint32_t cmd_flow_index; 2095 uint8_t cmd_reserved2[28]; 2096 } __packed __aligned(4); 2097 2098 struct mcx_cmd_query_flow_table_entry_out { 2099 uint8_t cmd_status; 2100 uint8_t cmd_reserved0[3]; 2101 uint32_t cmd_syndrome; 2102 uint8_t cmd_reserved1[8]; 2103 } __packed __aligned(4); 2104 2105 struct mcx_cmd_query_flow_table_entry_mb_out { 2106 uint8_t cmd_reserved0[48]; 2107 struct mcx_flow_ctx cmd_flow_ctx; 2108 } __packed __aligned(4); 2109 2110 struct mcx_cmd_delete_flow_table_entry_in { 2111 uint16_t cmd_opcode; 2112 uint8_t cmd_reserved0[4]; 2113 uint16_t cmd_op_mod; 2114 uint8_t cmd_reserved1[8]; 2115 } __packed __aligned(4); 2116 2117 struct mcx_cmd_delete_flow_table_entry_mb_in { 2118 uint8_t cmd_table_type; 2119 uint8_t cmd_reserved0[3]; 2120 uint32_t cmd_table_id; 2121 uint8_t cmd_reserved1[8]; 2122 uint32_t cmd_flow_index; 2123 uint8_t cmd_reserved2[28]; 2124 } __packed __aligned(4); 2125 2126 struct mcx_cmd_delete_flow_table_entry_out { 2127 uint8_t cmd_status; 2128 uint8_t cmd_reserved0[3]; 2129 uint32_t cmd_syndrome; 2130 uint8_t cmd_reserved1[8]; 2131 } __packed __aligned(4); 2132 2133 struct mcx_cmd_query_flow_group_in { 2134 uint16_t cmd_opcode; 2135 uint8_t cmd_reserved0[4]; 2136 uint16_t cmd_op_mod; 2137 uint8_t cmd_reserved1[8]; 2138 } __packed __aligned(4); 2139 2140 struct mcx_cmd_query_flow_group_mb_in { 2141 uint8_t cmd_table_type; 2142 uint8_t cmd_reserved0[3]; 2143 uint32_t cmd_table_id; 2144 uint32_t cmd_group_id; 2145 uint8_t cmd_reserved1[36]; 2146 } __packed __aligned(4); 2147 2148 struct mcx_cmd_query_flow_group_out { 2149 uint8_t cmd_status; 2150 uint8_t cmd_reserved0[3]; 2151 uint32_t cmd_syndrome; 2152 uint8_t cmd_reserved1[8]; 2153 } __packed __aligned(4); 2154 2155 struct mcx_cmd_query_flow_group_mb_out { 2156 uint8_t cmd_reserved0[12]; 2157 uint32_t cmd_start_flow_index; 2158 uint8_t cmd_reserved1[4]; 2159 uint32_t cmd_end_flow_index; 2160 uint8_t cmd_reserved2[20]; 2161 uint32_t cmd_match_criteria_enable; 2162 uint8_t cmd_match_criteria[512]; 2163 uint8_t cmd_reserved4[448]; 2164 } __packed __aligned(4); 2165 2166 struct mcx_cmd_query_flow_table_in { 2167 uint16_t cmd_opcode; 2168 uint8_t cmd_reserved0[4]; 2169 uint16_t cmd_op_mod; 2170 uint8_t cmd_reserved1[8]; 2171 } __packed __aligned(4); 2172 2173 struct mcx_cmd_query_flow_table_mb_in { 2174 uint8_t cmd_table_type; 2175 uint8_t cmd_reserved0[3]; 2176 uint32_t cmd_table_id; 2177 uint8_t cmd_reserved1[40]; 2178 } __packed __aligned(4); 2179 2180 struct mcx_cmd_query_flow_table_out { 2181 uint8_t cmd_status; 2182 uint8_t cmd_reserved0[3]; 2183 uint32_t cmd_syndrome; 2184 uint8_t cmd_reserved1[8]; 2185 } __packed __aligned(4); 2186 2187 struct mcx_cmd_query_flow_table_mb_out { 2188 uint8_t cmd_reserved0[4]; 2189 struct mcx_flow_table_ctx cmd_ctx; 2190 } __packed __aligned(4); 2191 2192 struct mcx_cmd_alloc_flow_counter_in { 2193 uint16_t cmd_opcode; 2194 uint8_t cmd_reserved0[4]; 2195 uint16_t cmd_op_mod; 2196 uint8_t cmd_reserved1[8]; 2197 } __packed __aligned(4); 2198 2199 struct mcx_cmd_query_rq_in { 2200 uint16_t cmd_opcode; 2201 uint8_t cmd_reserved0[4]; 2202 uint16_t cmd_op_mod; 2203 uint32_t cmd_rqn; 2204 uint8_t cmd_reserved1[4]; 2205 } __packed __aligned(4); 2206 2207 struct mcx_cmd_query_rq_out { 2208 uint8_t cmd_status; 2209 uint8_t cmd_reserved0[3]; 2210 uint32_t cmd_syndrome; 2211 uint8_t cmd_reserved1[8]; 2212 } __packed __aligned(4); 2213 2214 struct mcx_cmd_query_rq_mb_out { 2215 uint8_t cmd_reserved0[16]; 2216 struct mcx_rq_ctx cmd_ctx; 2217 }; 2218 2219 struct mcx_cmd_query_sq_in { 2220 uint16_t cmd_opcode; 2221 uint8_t cmd_reserved0[4]; 2222 uint16_t cmd_op_mod; 2223 uint32_t cmd_sqn; 2224 uint8_t cmd_reserved1[4]; 2225 } __packed __aligned(4); 2226 2227 struct mcx_cmd_query_sq_out { 2228 uint8_t cmd_status; 2229 uint8_t cmd_reserved0[3]; 2230 uint32_t cmd_syndrome; 2231 uint8_t cmd_reserved1[8]; 2232 } __packed __aligned(4); 2233 2234 struct mcx_cmd_query_sq_mb_out { 2235 uint8_t cmd_reserved0[16]; 2236 struct mcx_sq_ctx cmd_ctx; 2237 }; 2238 2239 struct mcx_cmd_alloc_flow_counter_out { 2240 uint8_t cmd_status; 2241 uint8_t cmd_reserved0[3]; 2242 uint32_t cmd_syndrome; 2243 uint8_t cmd_reserved1[2]; 2244 uint16_t cmd_flow_counter_id; 2245 uint8_t cmd_reserved2[4]; 2246 } __packed __aligned(4); 2247 2248 struct mcx_wq_doorbell { 2249 uint32_t db_recv_counter; 2250 uint32_t db_send_counter; 2251 } __packed __aligned(8); 2252 2253 struct mcx_dmamem { 2254 bus_dmamap_t mxm_map; 2255 bus_dma_segment_t mxm_seg; 2256 int mxm_nsegs; 2257 size_t mxm_size; 2258 void *mxm_kva; 2259 }; 2260 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2261 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2262 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2263 #define MCX_DMA_OFF(_mxm, _off) ((void *)((char *)(_mxm)->mxm_kva + (_off))) 2264 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2265 2266 struct mcx_hwmem { 2267 bus_dmamap_t mhm_map; 2268 bus_dma_segment_t *mhm_segs; 2269 unsigned int mhm_seg_count; 2270 unsigned int mhm_npages; 2271 }; 2272 2273 struct mcx_slot { 2274 bus_dmamap_t ms_map; 2275 struct mbuf *ms_m; 2276 }; 2277 2278 struct mcx_eq { 2279 int eq_n; 2280 uint32_t eq_cons; 2281 struct mcx_dmamem eq_mem; 2282 }; 2283 2284 struct mcx_cq { 2285 int cq_n; 2286 struct mcx_dmamem cq_mem; 2287 bus_addr_t cq_doorbell; 2288 uint32_t cq_cons; 2289 uint32_t cq_count; 2290 }; 2291 2292 struct mcx_calibration { 2293 uint64_t c_timestamp; /* previous mcx chip time */ 2294 uint64_t c_uptime; /* previous kernel nanouptime */ 2295 uint64_t c_tbase; /* mcx chip time */ 2296 uint64_t c_ubase; /* kernel nanouptime */ 2297 uint64_t c_ratio; 2298 }; 2299 2300 #define MCX_CALIBRATE_FIRST 2 2301 #define MCX_CALIBRATE_NORMAL 32 2302 2303 struct mcx_rxring { 2304 u_int rxr_total; 2305 u_int rxr_inuse; 2306 }; 2307 2308 MBUFQ_HEAD(mcx_mbufq); 2309 2310 struct mcx_rx { 2311 struct mcx_softc *rx_softc; 2312 2313 int rx_rqn; 2314 struct mcx_dmamem rx_rq_mem; 2315 struct mcx_slot *rx_slots; 2316 bus_addr_t rx_doorbell; 2317 2318 uint32_t rx_prod; 2319 callout_t rx_refill; 2320 struct mcx_rxring rx_rxr; 2321 } __aligned(64); 2322 2323 struct mcx_tx { 2324 struct mcx_softc *tx_softc; 2325 kmutex_t tx_lock; 2326 pcq_t *tx_pcq; 2327 void *tx_softint; 2328 2329 int tx_uar; 2330 int tx_sqn; 2331 struct mcx_dmamem tx_sq_mem; 2332 struct mcx_slot *tx_slots; 2333 bus_addr_t tx_doorbell; 2334 int tx_bf_offset; 2335 2336 uint32_t tx_cons; 2337 uint32_t tx_prod; 2338 } __aligned(64); 2339 2340 struct mcx_queues { 2341 void *q_ihc; 2342 struct mcx_softc *q_sc; 2343 int q_uar; 2344 int q_index; 2345 struct mcx_rx q_rx; 2346 struct mcx_tx q_tx; 2347 struct mcx_cq q_cq; 2348 struct mcx_eq q_eq; 2349 #if NKSTAT > 0 2350 struct kstat *q_kstat; 2351 #endif 2352 }; 2353 2354 struct mcx_flow_group { 2355 int g_id; 2356 int g_table; 2357 int g_start; 2358 int g_size; 2359 }; 2360 2361 #define MCX_FLOW_GROUP_PROMISC 0 2362 #define MCX_FLOW_GROUP_ALLMULTI 1 2363 #define MCX_FLOW_GROUP_MAC 2 2364 #define MCX_FLOW_GROUP_RSS_L4 3 2365 #define MCX_FLOW_GROUP_RSS_L3 4 2366 #define MCX_FLOW_GROUP_RSS_NONE 5 2367 #define MCX_NUM_FLOW_GROUPS 6 2368 2369 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2370 MCX_TIR_CTX_HASH_SEL_DST_IP 2371 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2372 MCX_TIR_CTX_HASH_SEL_DPORT 2373 2374 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2375 MCX_TIR_CTX_HASH_SEL_IPV4 2376 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2377 MCX_TIR_CTX_HASH_SEL_IPV6 2378 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2379 MCX_TIR_CTX_HASH_SEL_IPV4 2380 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2381 MCX_TIR_CTX_HASH_SEL_IPV6 2382 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2383 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2384 2385 /* 2386 * There are a few different pieces involved in configuring RSS. 2387 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2388 * different rx queues based on a hash value. We only create one, because 2389 * we want to scatter any traffic we can apply RSS to across all our rx 2390 * queues. Anything else will only be delivered to the first rx queue, 2391 * which doesn't require an RQT. 2392 * 2393 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2394 * queue or an RQT, and in the latter case, specifies the set of fields 2395 * hashed, the hash function, and the hash key. We need one of these for each 2396 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2397 * and one for non-RSS traffic. 2398 * 2399 * Flow tables hold flow table entries in sequence. The first entry that 2400 * matches a packet is applied, sending the packet to either another flow 2401 * table or a TIR. We use one flow table to select packets based on 2402 * destination MAC address, and a second to apply RSS. The entries in the 2403 * first table send matching packets to the second, and the entries in the 2404 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2405 * 2406 * The flow table entry that delivers packets to an RSS TIR must include match 2407 * criteria that ensure packets delivered to the TIR include all the fields 2408 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2409 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2410 * each TIR. 2411 * 2412 * All of this is a lot more flexible than we need, and we can describe most 2413 * of the stuff we need with a simple array. 2414 * 2415 * An RSS config creates a TIR with hashing enabled on a set of fields, 2416 * pointing to either the first rx queue or the RQT containing all the rx 2417 * queues, and a flow table entry that matches on an ether type and 2418 * optionally an ip proto, that delivers packets to the TIR. 2419 */ 2420 static struct mcx_rss_rule { 2421 int hash_sel; 2422 int flow_group; 2423 int ethertype; 2424 int ip_proto; 2425 } mcx_rss_config[] = { 2426 /* udp and tcp for v4/v6 */ 2427 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2428 ETHERTYPE_IP, IPPROTO_TCP }, 2429 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2430 ETHERTYPE_IPV6, IPPROTO_TCP }, 2431 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2432 ETHERTYPE_IP, IPPROTO_UDP }, 2433 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2434 ETHERTYPE_IPV6, IPPROTO_UDP }, 2435 2436 /* other v4/v6 */ 2437 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2438 ETHERTYPE_IP, 0 }, 2439 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2440 ETHERTYPE_IPV6, 0 }, 2441 2442 /* non v4/v6 */ 2443 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2444 }; 2445 2446 struct mcx_softc { 2447 device_t sc_dev; 2448 struct ethercom sc_ec; 2449 struct ifmedia sc_media; 2450 uint64_t sc_media_status; 2451 uint64_t sc_media_active; 2452 kmutex_t sc_media_mutex; 2453 2454 pci_chipset_tag_t sc_pc; 2455 pci_intr_handle_t *sc_intrs; 2456 void *sc_ihc; 2457 pcitag_t sc_tag; 2458 2459 bus_dma_tag_t sc_dmat; 2460 bus_space_tag_t sc_memt; 2461 bus_space_handle_t sc_memh; 2462 bus_size_t sc_mems; 2463 2464 struct mcx_dmamem sc_cmdq_mem; 2465 unsigned int sc_cmdq_mask; 2466 unsigned int sc_cmdq_size; 2467 2468 unsigned int sc_cmdq_token; 2469 2470 struct mcx_hwmem sc_boot_pages; 2471 struct mcx_hwmem sc_init_pages; 2472 struct mcx_hwmem sc_regular_pages; 2473 2474 int sc_uar; 2475 int sc_pd; 2476 int sc_tdomain; 2477 uint32_t sc_lkey; 2478 int sc_tis; 2479 int sc_tir[__arraycount(mcx_rss_config)]; 2480 int sc_rqt; 2481 2482 struct mcx_dmamem sc_doorbell_mem; 2483 2484 struct mcx_eq sc_admin_eq; 2485 struct mcx_eq sc_queue_eq; 2486 2487 int sc_hardmtu; 2488 int sc_rxbufsz; 2489 2490 int sc_bf_size; 2491 int sc_max_rqt_size; 2492 2493 struct workqueue *sc_workq; 2494 struct work sc_port_change; 2495 2496 int sc_mac_flow_table_id; 2497 int sc_rss_flow_table_id; 2498 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2499 int sc_promisc_flow_enabled; 2500 int sc_allmulti_flow_enabled; 2501 int sc_mcast_flow_base; 2502 int sc_extra_mcast; 2503 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2504 2505 struct mcx_calibration sc_calibration[2]; 2506 unsigned int sc_calibration_gen; 2507 callout_t sc_calibrate; 2508 uint32_t sc_mhz; 2509 uint32_t sc_khz; 2510 2511 struct mcx_queues *sc_queues; 2512 unsigned int sc_nqueues; 2513 2514 int sc_mcam_reg; 2515 2516 #if NKSTAT > 0 2517 struct kstat *sc_kstat_ieee8023; 2518 struct kstat *sc_kstat_rfc2863; 2519 struct kstat *sc_kstat_rfc2819; 2520 struct kstat *sc_kstat_rfc3635; 2521 unsigned int sc_kstat_mtmp_count; 2522 struct kstat **sc_kstat_mtmp; 2523 #endif 2524 2525 struct timecounter sc_timecounter; 2526 }; 2527 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 2528 2529 static int mcx_match(device_t, cfdata_t, void *); 2530 static void mcx_attach(device_t, device_t, void *); 2531 2532 static void * mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *, 2533 int (*)(void *), void *, const char *); 2534 2535 static void mcx_rxr_init(struct mcx_rxring *, u_int, u_int); 2536 static u_int mcx_rxr_get(struct mcx_rxring *, u_int); 2537 static void mcx_rxr_put(struct mcx_rxring *, u_int); 2538 static u_int mcx_rxr_inuse(struct mcx_rxring *); 2539 2540 #if NKSTAT > 0 2541 static void mcx_kstat_attach(struct mcx_softc *); 2542 #endif 2543 2544 static void mcx_timecounter_attach(struct mcx_softc *); 2545 2546 static int mcx_version(struct mcx_softc *); 2547 static int mcx_init_wait(struct mcx_softc *); 2548 static int mcx_enable_hca(struct mcx_softc *); 2549 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2550 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2551 int); 2552 static int mcx_issi(struct mcx_softc *); 2553 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2554 static int mcx_hca_max_caps(struct mcx_softc *); 2555 static int mcx_hca_set_caps(struct mcx_softc *); 2556 static int mcx_init_hca(struct mcx_softc *); 2557 static int mcx_set_driver_version(struct mcx_softc *); 2558 static int mcx_iff(struct mcx_softc *); 2559 static int mcx_alloc_uar(struct mcx_softc *, int *); 2560 static int mcx_alloc_pd(struct mcx_softc *); 2561 static int mcx_alloc_tdomain(struct mcx_softc *); 2562 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2563 uint64_t, int); 2564 static int mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *); 2565 static int mcx_query_special_contexts(struct mcx_softc *); 2566 static int mcx_set_port_mtu(struct mcx_softc *, int); 2567 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2568 int); 2569 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2570 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2571 int); 2572 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2573 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2574 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2575 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2576 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2577 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2578 int *); 2579 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2580 int *); 2581 static int mcx_destroy_tir(struct mcx_softc *, int); 2582 static int mcx_create_tis(struct mcx_softc *, int *); 2583 static int mcx_destroy_tis(struct mcx_softc *, int); 2584 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2585 static int mcx_destroy_rqt(struct mcx_softc *, int); 2586 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2587 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2588 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2589 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2590 int, int, struct mcx_flow_match *); 2591 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2592 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2593 const uint8_t *, uint32_t); 2594 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2595 int, int, uint32_t); 2596 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2597 2598 #if NKSTAT > 0 2599 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *); 2600 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *); 2601 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *); 2602 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *); 2603 #endif 2604 2605 #if 0 2606 static int mcx_dump_flow_table(struct mcx_softc *, int); 2607 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2608 static int mcx_dump_flow_group(struct mcx_softc *, int); 2609 #endif 2610 2611 2612 /* 2613 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2614 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2615 */ 2616 static void mcx_refill(void *); 2617 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2618 struct mcx_cq_entry *, struct mcx_mbufq *, 2619 const struct mcx_calibration *); 2620 static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2621 struct mcx_cq_entry *); 2622 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2623 struct mcx_cq *); 2624 2625 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2626 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2627 static int mcx_admin_intr(void *); 2628 static int mcx_cq_intr(void *); 2629 2630 static int mcx_init(struct ifnet *); 2631 static void mcx_stop(struct ifnet *, int); 2632 static int mcx_ioctl(struct ifnet *, u_long, void *); 2633 static void mcx_start(struct ifnet *); 2634 static int mcx_transmit(struct ifnet *, struct mbuf *); 2635 static void mcx_deferred_transmit(void *); 2636 static void mcx_watchdog(struct ifnet *); 2637 static void mcx_media_add_types(struct mcx_softc *); 2638 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2639 static int mcx_media_change(struct ifnet *); 2640 #if 0 2641 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2642 #endif 2643 static void mcx_port_change(struct work *, void *); 2644 2645 static void mcx_calibrate_first(struct mcx_softc *); 2646 static void mcx_calibrate(void *); 2647 2648 static inline uint32_t 2649 mcx_rd(struct mcx_softc *, bus_size_t); 2650 static inline void 2651 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2652 static inline void 2653 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2654 2655 static uint64_t mcx_timer(struct mcx_softc *); 2656 2657 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2658 bus_size_t, u_int align); 2659 static void mcx_dmamem_zero(struct mcx_dmamem *); 2660 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2661 2662 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2663 unsigned int); 2664 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2665 2666 CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL); 2667 2668 static const struct { 2669 pci_vendor_id_t vendor; 2670 pci_product_id_t product; 2671 } mcx_devices[] = { 2672 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2673 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2674 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2675 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2676 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2677 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2678 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2679 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2680 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 }, 2681 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2682 }; 2683 2684 struct mcx_eth_proto_capability { 2685 uint64_t cap_media; 2686 uint64_t cap_baudrate; 2687 }; 2688 2689 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2690 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2691 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2692 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2693 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2694 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2695 [MCX_ETHER_CAP_20G_KR2] = { IFM_20G_KR2, IF_Gbps(20) }, 2696 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2697 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2698 [MCX_ETHER_CAP_56G_R4] = { IFM_56G_R4, IF_Gbps(56) }, 2699 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_CR1, IF_Gbps(10) }, 2700 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2701 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2702 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2703 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2704 [MCX_ETHER_CAP_50G_SR2] = { IFM_50G_SR2, IF_Gbps(50) }, 2705 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2706 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2707 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2708 [MCX_ETHER_CAP_100G_LR4] = { IFM_100G_LR4, IF_Gbps(100) }, 2709 [MCX_ETHER_CAP_100_TX] = { IFM_100_TX, IF_Mbps(100) }, 2710 [MCX_ETHER_CAP_1000_T] = { IFM_1000_T, IF_Gbps(1) }, 2711 [MCX_ETHER_CAP_10G_T] = { IFM_10G_T, IF_Gbps(10) }, 2712 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2713 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2714 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2715 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2716 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2717 }; 2718 2719 static int 2720 mcx_get_id(uint32_t val) 2721 { 2722 return be32toh(val) & 0x00ffffff; 2723 } 2724 2725 static int 2726 mcx_match(device_t parent, cfdata_t cf, void *aux) 2727 { 2728 struct pci_attach_args *pa = aux; 2729 int n; 2730 2731 for (n = 0; n < __arraycount(mcx_devices); n++) { 2732 if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor && 2733 PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product) 2734 return 1; 2735 } 2736 2737 return 0; 2738 } 2739 2740 void 2741 mcx_attach(device_t parent, device_t self, void *aux) 2742 { 2743 struct mcx_softc *sc = device_private(self); 2744 struct ifnet *ifp = &sc->sc_ec.ec_if; 2745 struct pci_attach_args *pa = aux; 2746 struct ifcapreq ifcr; 2747 uint8_t enaddr[ETHER_ADDR_LEN]; 2748 int counts[PCI_INTR_TYPE_SIZE]; 2749 char intrxname[32]; 2750 pcireg_t memtype; 2751 uint32_t r; 2752 unsigned int cq_stride; 2753 unsigned int cq_size; 2754 int i, msix; 2755 kcpuset_t *affinity; 2756 2757 sc->sc_dev = self; 2758 sc->sc_pc = pa->pa_pc; 2759 sc->sc_tag = pa->pa_tag; 2760 if (pci_dma64_available(pa)) 2761 sc->sc_dmat = pa->pa_dmat64; 2762 else 2763 sc->sc_dmat = pa->pa_dmat; 2764 2765 /* Map the PCI memory space */ 2766 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2767 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2768 #ifdef __NetBSD__ 2769 0, 2770 #else 2771 BUS_SPACE_MAP_PREFETCHABLE, 2772 #endif 2773 &sc->sc_memt, &sc->sc_memh, 2774 NULL, &sc->sc_mems)) { 2775 aprint_error(": unable to map register memory\n"); 2776 return; 2777 } 2778 2779 pci_aprint_devinfo(pa, "Ethernet controller"); 2780 2781 mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET); 2782 2783 if (mcx_version(sc) != 0) { 2784 /* error printed by mcx_version */ 2785 goto unmap; 2786 } 2787 2788 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2789 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2790 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2791 if (cq_size > MCX_MAX_CQE) { 2792 aprint_error_dev(self, 2793 "command queue size overflow %u\n", cq_size); 2794 goto unmap; 2795 } 2796 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2797 aprint_error_dev(self, 2798 "command queue entry size underflow %u\n", cq_stride); 2799 goto unmap; 2800 } 2801 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2802 aprint_error_dev(self, "command queue page overflow\n"); 2803 goto unmap; 2804 } 2805 2806 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2807 MCX_PAGE_SIZE) != 0) { 2808 aprint_error_dev(self, "unable to allocate doorbell memory\n"); 2809 goto unmap; 2810 } 2811 2812 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2813 MCX_PAGE_SIZE) != 0) { 2814 aprint_error_dev(self, "unable to allocate command queue\n"); 2815 goto dbfree; 2816 } 2817 2818 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2819 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2820 BUS_SPACE_BARRIER_WRITE); 2821 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2822 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2823 BUS_SPACE_BARRIER_WRITE); 2824 2825 if (mcx_init_wait(sc) != 0) { 2826 aprint_error_dev(self, "timeout waiting for init\n"); 2827 goto cqfree; 2828 } 2829 2830 sc->sc_cmdq_mask = cq_size - 1; 2831 sc->sc_cmdq_size = cq_stride; 2832 2833 if (mcx_enable_hca(sc) != 0) { 2834 /* error printed by mcx_enable_hca */ 2835 goto cqfree; 2836 } 2837 2838 if (mcx_issi(sc) != 0) { 2839 /* error printed by mcx_issi */ 2840 goto teardown; 2841 } 2842 2843 if (mcx_pages(sc, &sc->sc_boot_pages, 2844 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2845 /* error printed by mcx_pages */ 2846 goto teardown; 2847 } 2848 2849 if (mcx_hca_max_caps(sc) != 0) { 2850 /* error printed by mcx_hca_max_caps */ 2851 goto teardown; 2852 } 2853 2854 if (mcx_hca_set_caps(sc) != 0) { 2855 /* error printed by mcx_hca_set_caps */ 2856 goto teardown; 2857 } 2858 2859 if (mcx_pages(sc, &sc->sc_init_pages, 2860 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2861 /* error printed by mcx_pages */ 2862 goto teardown; 2863 } 2864 2865 if (mcx_init_hca(sc) != 0) { 2866 /* error printed by mcx_init_hca */ 2867 goto teardown; 2868 } 2869 2870 if (mcx_pages(sc, &sc->sc_regular_pages, 2871 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2872 /* error printed by mcx_pages */ 2873 goto teardown; 2874 } 2875 2876 /* apparently not necessary? */ 2877 if (mcx_set_driver_version(sc) != 0) { 2878 /* error printed by mcx_set_driver_version */ 2879 goto teardown; 2880 } 2881 2882 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2883 /* error printed by mcx_iff? */ 2884 goto teardown; 2885 } 2886 2887 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2888 /* error printed by mcx_alloc_uar */ 2889 goto teardown; 2890 } 2891 2892 if (mcx_alloc_pd(sc) != 0) { 2893 /* error printed by mcx_alloc_pd */ 2894 goto teardown; 2895 } 2896 2897 if (mcx_alloc_tdomain(sc) != 0) { 2898 /* error printed by mcx_alloc_tdomain */ 2899 goto teardown; 2900 } 2901 2902 /* 2903 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2904 * mellanox support tells me legacy interrupts are not supported, 2905 * so we're stuck with just msi-x. 2906 */ 2907 counts[PCI_INTR_TYPE_MSIX] = -1; 2908 counts[PCI_INTR_TYPE_MSI] = 0; 2909 counts[PCI_INTR_TYPE_INTX] = 0; 2910 if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) { 2911 aprint_error_dev(self, "unable to allocate interrupt\n"); 2912 goto teardown; 2913 } 2914 if (counts[PCI_INTR_TYPE_MSIX] < 2) { 2915 aprint_error_dev(self, "not enough MSI-X vectors\n"); 2916 goto teardown; 2917 } 2918 KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX); 2919 snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc)); 2920 sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc, 2921 intrxname); 2922 if (sc->sc_ihc == NULL) { 2923 aprint_error_dev(self, "couldn't establish adminq interrupt\n"); 2924 goto teardown; 2925 } 2926 2927 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2928 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2929 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2930 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2931 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2932 /* error printed by mcx_create_eq */ 2933 goto teardown; 2934 } 2935 2936 if (mcx_query_nic_vport_context(sc, enaddr) != 0) { 2937 /* error printed by mcx_query_nic_vport_context */ 2938 goto teardown; 2939 } 2940 2941 if (mcx_query_special_contexts(sc) != 0) { 2942 /* error printed by mcx_query_special_contexts */ 2943 goto teardown; 2944 } 2945 2946 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2947 /* error printed by mcx_set_port_mtu */ 2948 goto teardown; 2949 } 2950 2951 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 2952 ether_sprintf(enaddr)); 2953 2954 msix = counts[PCI_INTR_TYPE_MSIX]; 2955 msix--; /* admin ops took one */ 2956 2957 sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix); 2958 sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu); 2959 sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues), 2960 KM_SLEEP); 2961 2962 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2963 ifp->if_softc = sc; 2964 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2965 #ifdef MCX_MPSAFE 2966 ifp->if_extflags = IFEF_MPSAFE; 2967 #endif 2968 ifp->if_init = mcx_init; 2969 ifp->if_stop = mcx_stop; 2970 ifp->if_ioctl = mcx_ioctl; 2971 ifp->if_start = mcx_start; 2972 if (sc->sc_nqueues > 1) { 2973 ifp->if_transmit = mcx_transmit; 2974 } 2975 ifp->if_watchdog = mcx_watchdog; 2976 ifp->if_mtu = sc->sc_hardmtu; 2977 ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx | 2978 IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx | 2979 IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx | 2980 IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx | 2981 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx; 2982 IFQ_SET_MAXLEN(&ifp->if_snd, 1024); 2983 IFQ_SET_READY(&ifp->if_snd); 2984 2985 sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU | 2986 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING; 2987 sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; 2988 2989 sc->sc_ec.ec_ifmedia = &sc->sc_media; 2990 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change, 2991 mcx_media_status, &sc->sc_media_mutex); 2992 mcx_media_add_types(sc); 2993 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2994 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2995 2996 if_attach(ifp); 2997 2998 /* Enable hardware offload by default */ 2999 memset(&ifcr, 0, sizeof(ifcr)); 3000 ifcr.ifcr_capenable = ifp->if_capabilities; 3001 ifioctl_common(ifp, SIOCSIFCAP, &ifcr); 3002 3003 if_deferred_start_init(ifp, NULL); 3004 3005 ether_ifattach(ifp, enaddr); 3006 3007 kcpuset_create(&affinity, false); 3008 kcpuset_set(affinity, 0); 3009 3010 for (i = 0; i < sc->sc_nqueues; i++) { 3011 struct mcx_queues *q = &sc->sc_queues[i]; 3012 struct mcx_rx *rx = &q->q_rx; 3013 struct mcx_tx *tx = &q->q_tx; 3014 int vec; 3015 3016 vec = i + 1; 3017 q->q_sc = sc; 3018 q->q_index = i; 3019 3020 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 3021 aprint_error_dev(self, "unable to alloc uar %d\n", i); 3022 goto teardown; 3023 } 3024 3025 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 3026 aprint_error_dev(self, 3027 "unable to create event queue %d\n", i); 3028 goto teardown; 3029 } 3030 3031 rx->rx_softc = sc; 3032 callout_init(&rx->rx_refill, CALLOUT_FLAGS); 3033 callout_setfunc(&rx->rx_refill, mcx_refill, rx); 3034 3035 tx->tx_softc = sc; 3036 mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET); 3037 tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP); 3038 tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, 3039 mcx_deferred_transmit, tx); 3040 3041 snprintf(intrxname, sizeof(intrxname), "%s queue %d", 3042 DEVNAME(sc), i); 3043 q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr, 3044 q, intrxname); 3045 } 3046 3047 callout_init(&sc->sc_calibrate, CALLOUT_FLAGS); 3048 callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc); 3049 3050 if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc, 3051 PRI_NONE, IPL_NET, 0) != 0) { 3052 aprint_error_dev(self, "couldn't create port change workq\n"); 3053 goto teardown; 3054 } 3055 3056 mcx_port_change(&sc->sc_port_change, sc); 3057 3058 sc->sc_mac_flow_table_id = -1; 3059 sc->sc_rss_flow_table_id = -1; 3060 sc->sc_rqt = -1; 3061 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 3062 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 3063 mfg->g_id = -1; 3064 mfg->g_table = -1; 3065 mfg->g_size = 0; 3066 mfg->g_start = 0; 3067 } 3068 sc->sc_extra_mcast = 0; 3069 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 3070 3071 #if NKSTAT > 0 3072 mcx_kstat_attach(sc); 3073 #endif 3074 mcx_timecounter_attach(sc); 3075 return; 3076 3077 teardown: 3078 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 3079 /* error printed by mcx_teardown_hca, and we're already unwinding */ 3080 cqfree: 3081 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 3082 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 3083 BUS_SPACE_BARRIER_WRITE); 3084 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 3085 MCX_CMDQ_INTERFACE_DISABLED); 3086 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 3087 BUS_SPACE_BARRIER_WRITE); 3088 3089 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 3090 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 3091 BUS_SPACE_BARRIER_WRITE); 3092 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 3093 3094 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 3095 dbfree: 3096 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 3097 unmap: 3098 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 3099 sc->sc_mems = 0; 3100 } 3101 3102 static void * 3103 mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity, 3104 int (*func)(void *), void *arg, const char *xname) 3105 { 3106 char intrbuf[PCI_INTRSTR_LEN]; 3107 const char *intrstr; 3108 void *ih; 3109 3110 pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE, 3111 true); 3112 3113 intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf, 3114 sizeof(intrbuf)); 3115 ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET, 3116 func, arg, xname); 3117 if (ih == NULL) { 3118 aprint_error_dev(sc->sc_dev, 3119 "unable to establish interrupt%s%s\n", 3120 intrstr ? " at " : "", 3121 intrstr ? intrstr : ""); 3122 return NULL; 3123 } 3124 3125 if (affinity != NULL && index > 0) { 3126 /* Round-robin affinity */ 3127 kcpuset_zero(affinity); 3128 kcpuset_set(affinity, (index - 1) % ncpu); 3129 interrupt_distribute(ih, affinity, NULL); 3130 } 3131 3132 return ih; 3133 } 3134 3135 static void 3136 mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm) 3137 { 3138 rxr->rxr_total = hwm; 3139 rxr->rxr_inuse = 0; 3140 } 3141 3142 static u_int 3143 mcx_rxr_get(struct mcx_rxring *rxr, u_int max) 3144 { 3145 const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse); 3146 3147 rxr->rxr_inuse += taken; 3148 3149 return taken; 3150 } 3151 3152 static void 3153 mcx_rxr_put(struct mcx_rxring *rxr, u_int n) 3154 { 3155 rxr->rxr_inuse -= n; 3156 } 3157 3158 static u_int 3159 mcx_rxr_inuse(struct mcx_rxring *rxr) 3160 { 3161 return rxr->rxr_inuse; 3162 } 3163 3164 static int 3165 mcx_version(struct mcx_softc *sc) 3166 { 3167 uint32_t fw0, fw1; 3168 uint16_t cmdif; 3169 3170 fw0 = mcx_rd(sc, MCX_FW_VER); 3171 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 3172 3173 aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0), 3174 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 3175 3176 cmdif = MCX_CMDIF(fw1); 3177 if (cmdif != MCX_CMD_IF_SUPPORTED) { 3178 aprint_error_dev(sc->sc_dev, 3179 "unsupported command interface %u\n", cmdif); 3180 return (-1); 3181 } 3182 3183 return (0); 3184 } 3185 3186 static int 3187 mcx_init_wait(struct mcx_softc *sc) 3188 { 3189 unsigned int i; 3190 uint32_t r; 3191 3192 for (i = 0; i < 2000; i++) { 3193 r = mcx_rd(sc, MCX_STATE); 3194 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 3195 return (0); 3196 3197 delay(1000); 3198 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 3199 BUS_SPACE_BARRIER_READ); 3200 } 3201 3202 return (-1); 3203 } 3204 3205 static uint8_t 3206 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3207 unsigned int msec) 3208 { 3209 unsigned int i; 3210 3211 for (i = 0; i < msec; i++) { 3212 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3213 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 3214 3215 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 3216 MCX_CQ_STATUS_OWN_SW) 3217 return (0); 3218 3219 delay(1000); 3220 } 3221 3222 return (ETIMEDOUT); 3223 } 3224 3225 static uint32_t 3226 mcx_mix_u64(uint32_t xor, uint64_t u64) 3227 { 3228 xor ^= u64 >> 32; 3229 xor ^= u64; 3230 3231 return (xor); 3232 } 3233 3234 static uint32_t 3235 mcx_mix_u32(uint32_t xor, uint32_t u32) 3236 { 3237 xor ^= u32; 3238 3239 return (xor); 3240 } 3241 3242 static uint32_t 3243 mcx_mix_u8(uint32_t xor, uint8_t u8) 3244 { 3245 xor ^= u8; 3246 3247 return (xor); 3248 } 3249 3250 static uint8_t 3251 mcx_mix_done(uint32_t xor) 3252 { 3253 xor ^= xor >> 16; 3254 xor ^= xor >> 8; 3255 3256 return (xor); 3257 } 3258 3259 static uint8_t 3260 mcx_xor(const void *buf, size_t len) 3261 { 3262 const uint32_t *dwords = buf; 3263 uint32_t xor = 0xff; 3264 size_t i; 3265 3266 len /= sizeof(*dwords); 3267 3268 for (i = 0; i < len; i++) 3269 xor ^= dwords[i]; 3270 3271 return (mcx_mix_done(xor)); 3272 } 3273 3274 static uint8_t 3275 mcx_cmdq_token(struct mcx_softc *sc) 3276 { 3277 uint8_t token; 3278 3279 do { 3280 token = ++sc->sc_cmdq_token; 3281 } while (token == 0); 3282 3283 return (token); 3284 } 3285 3286 static void 3287 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3288 uint32_t ilen, uint32_t olen, uint8_t token) 3289 { 3290 memset(cqe, 0, sc->sc_cmdq_size); 3291 3292 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 3293 be32enc(&cqe->cq_input_length, ilen); 3294 be32enc(&cqe->cq_output_length, olen); 3295 cqe->cq_token = token; 3296 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 3297 } 3298 3299 static void 3300 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 3301 { 3302 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 3303 } 3304 3305 static int 3306 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 3307 { 3308 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 3309 return (0); 3310 } 3311 3312 static void * 3313 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 3314 { 3315 return (&cqe->cq_input_data); 3316 } 3317 3318 static void * 3319 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3320 { 3321 return (&cqe->cq_output_data); 3322 } 3323 3324 static void 3325 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3326 unsigned int slot) 3327 { 3328 mcx_cmdq_sign(cqe); 3329 3330 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3331 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3332 3333 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3334 mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t), 3335 BUS_SPACE_BARRIER_WRITE); 3336 } 3337 3338 static int 3339 mcx_enable_hca(struct mcx_softc *sc) 3340 { 3341 struct mcx_cmdq_entry *cqe; 3342 struct mcx_cmd_enable_hca_in *in; 3343 struct mcx_cmd_enable_hca_out *out; 3344 int error; 3345 uint8_t status; 3346 3347 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3348 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3349 3350 in = mcx_cmdq_in(cqe); 3351 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3352 in->cmd_op_mod = htobe16(0); 3353 in->cmd_function_id = htobe16(0); 3354 3355 mcx_cmdq_post(sc, cqe, 0); 3356 3357 error = mcx_cmdq_poll(sc, cqe, 1000); 3358 if (error != 0) { 3359 printf(", hca enable timeout\n"); 3360 return (-1); 3361 } 3362 if (mcx_cmdq_verify(cqe) != 0) { 3363 printf(", hca enable command corrupt\n"); 3364 return (-1); 3365 } 3366 3367 status = cqe->cq_output_data[0]; 3368 if (status != MCX_CQ_STATUS_OK) { 3369 printf(", hca enable failed (%x)\n", status); 3370 return (-1); 3371 } 3372 3373 return (0); 3374 } 3375 3376 static int 3377 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3378 { 3379 struct mcx_cmdq_entry *cqe; 3380 struct mcx_cmd_teardown_hca_in *in; 3381 struct mcx_cmd_teardown_hca_out *out; 3382 int error; 3383 uint8_t status; 3384 3385 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3386 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3387 3388 in = mcx_cmdq_in(cqe); 3389 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3390 in->cmd_op_mod = htobe16(0); 3391 in->cmd_profile = profile; 3392 3393 mcx_cmdq_post(sc, cqe, 0); 3394 3395 error = mcx_cmdq_poll(sc, cqe, 1000); 3396 if (error != 0) { 3397 printf(", hca teardown timeout\n"); 3398 return (-1); 3399 } 3400 if (mcx_cmdq_verify(cqe) != 0) { 3401 printf(", hca teardown command corrupt\n"); 3402 return (-1); 3403 } 3404 3405 status = cqe->cq_output_data[0]; 3406 if (status != MCX_CQ_STATUS_OK) { 3407 printf(", hca teardown failed (%x)\n", status); 3408 return (-1); 3409 } 3410 3411 return (0); 3412 } 3413 3414 static int 3415 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3416 unsigned int nmb, uint64_t *ptr, uint8_t token) 3417 { 3418 uint8_t *kva; 3419 uint64_t dva; 3420 int i; 3421 int error; 3422 3423 error = mcx_dmamem_alloc(sc, mxm, 3424 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3425 if (error != 0) 3426 return (error); 3427 3428 mcx_dmamem_zero(mxm); 3429 3430 dva = MCX_DMA_DVA(mxm); 3431 kva = MCX_DMA_KVA(mxm); 3432 for (i = 0; i < nmb; i++) { 3433 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3434 3435 /* patch the cqe or mbox pointing at this one */ 3436 be64enc(ptr, dva); 3437 3438 /* fill in this mbox */ 3439 be32enc(&mbox->mb_block_number, i); 3440 mbox->mb_token = token; 3441 3442 /* move to the next one */ 3443 ptr = &mbox->mb_next_ptr; 3444 3445 dva += MCX_CMDQ_MAILBOX_SIZE; 3446 kva += MCX_CMDQ_MAILBOX_SIZE; 3447 } 3448 3449 return (0); 3450 } 3451 3452 static uint32_t 3453 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3454 { 3455 uint32_t xor = 0xff; 3456 3457 /* only 3 fields get set, so mix them directly */ 3458 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3459 xor = mcx_mix_u32(xor, mb->mb_block_number); 3460 xor = mcx_mix_u8(xor, mb->mb_token); 3461 3462 return (mcx_mix_done(xor)); 3463 } 3464 3465 static void 3466 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3467 { 3468 uint8_t *kva; 3469 int i; 3470 3471 kva = MCX_DMA_KVA(mxm); 3472 3473 for (i = 0; i < nmb; i++) { 3474 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3475 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3476 mb->mb_ctrl_signature = sig; 3477 mb->mb_signature = sig ^ 3478 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3479 3480 kva += MCX_CMDQ_MAILBOX_SIZE; 3481 } 3482 } 3483 3484 static void 3485 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3486 { 3487 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3488 0, MCX_DMA_LEN(mxm), ops); 3489 } 3490 3491 static struct mcx_cmdq_mailbox * 3492 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3493 { 3494 uint8_t *kva; 3495 3496 kva = MCX_DMA_KVA(mxm); 3497 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3498 3499 return ((struct mcx_cmdq_mailbox *)kva); 3500 } 3501 3502 static inline void * 3503 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3504 { 3505 return (&mb->mb_data); 3506 } 3507 3508 static void 3509 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3510 void *b, size_t len) 3511 { 3512 uint8_t *buf = b; 3513 struct mcx_cmdq_mailbox *mb; 3514 int i; 3515 3516 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3517 for (i = 0; i < nmb; i++) { 3518 3519 memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len)); 3520 3521 if (sizeof(mb->mb_data) >= len) 3522 break; 3523 3524 buf += sizeof(mb->mb_data); 3525 len -= sizeof(mb->mb_data); 3526 mb++; 3527 } 3528 } 3529 3530 static void 3531 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3532 struct mcx_dmamem *buf) 3533 { 3534 uint64_t *pas; 3535 int mbox, mbox_pages, i; 3536 3537 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3538 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3539 3540 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3541 pas += (offset / sizeof(*pas)); 3542 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3543 for (i = 0; i < npages; i++) { 3544 if (i == mbox_pages) { 3545 mbox++; 3546 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3547 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3548 } 3549 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3550 pas++; 3551 } 3552 } 3553 3554 static void 3555 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3556 { 3557 uint8_t *buf = b; 3558 struct mcx_cmdq_mailbox *mb; 3559 int i; 3560 3561 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3562 for (i = 0; i < nmb; i++) { 3563 memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len)); 3564 3565 if (sizeof(mb->mb_data) >= len) 3566 break; 3567 3568 buf += sizeof(mb->mb_data); 3569 len -= sizeof(mb->mb_data); 3570 mb++; 3571 } 3572 } 3573 3574 static void 3575 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3576 { 3577 mcx_dmamem_free(sc, mxm); 3578 } 3579 3580 #if 0 3581 static void 3582 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3583 { 3584 unsigned int i; 3585 3586 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3587 be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr)); 3588 3589 printf(", idata "); 3590 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3591 printf("%02x", cqe->cq_input_data[i]); 3592 3593 printf(", odata "); 3594 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3595 printf("%02x", cqe->cq_output_data[i]); 3596 3597 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3598 be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length), 3599 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3600 } 3601 3602 static void 3603 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3604 { 3605 int i, j; 3606 uint8_t *d; 3607 3608 for (i = 0; i < num; i++) { 3609 struct mcx_cmdq_mailbox *mbox; 3610 mbox = mcx_cq_mbox(mboxes, i); 3611 3612 d = mcx_cq_mbox_data(mbox); 3613 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3614 if (j != 0 && (j % 16 == 0)) 3615 printf("\n"); 3616 printf("%.2x ", d[j]); 3617 } 3618 } 3619 } 3620 #endif 3621 3622 static int 3623 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3624 int len) 3625 { 3626 struct mcx_dmamem mxm; 3627 struct mcx_cmdq_entry *cqe; 3628 struct mcx_cmd_access_reg_in *in; 3629 struct mcx_cmd_access_reg_out *out; 3630 uint8_t token = mcx_cmdq_token(sc); 3631 int error, nmb; 3632 3633 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3634 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3635 token); 3636 3637 in = mcx_cmdq_in(cqe); 3638 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3639 in->cmd_op_mod = htobe16(op); 3640 in->cmd_register_id = htobe16(reg); 3641 3642 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3643 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3644 &cqe->cq_output_ptr, token) != 0) { 3645 printf(", unable to allocate access reg mailboxen\n"); 3646 return (-1); 3647 } 3648 cqe->cq_input_ptr = cqe->cq_output_ptr; 3649 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3650 mcx_cmdq_mboxes_sign(&mxm, nmb); 3651 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3652 3653 mcx_cmdq_post(sc, cqe, 0); 3654 error = mcx_cmdq_poll(sc, cqe, 1000); 3655 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3656 3657 if (error != 0) { 3658 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3659 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3660 goto free; 3661 } 3662 error = mcx_cmdq_verify(cqe); 3663 if (error != 0) { 3664 printf("%s: access reg (%s %x) reply corrupt\n", 3665 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3666 reg); 3667 goto free; 3668 } 3669 3670 out = mcx_cmdq_out(cqe); 3671 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3672 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3673 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3674 reg, out->cmd_status, be32toh(out->cmd_syndrome)); 3675 error = -1; 3676 goto free; 3677 } 3678 3679 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3680 free: 3681 mcx_dmamem_free(sc, &mxm); 3682 3683 return (error); 3684 } 3685 3686 static int 3687 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3688 unsigned int slot) 3689 { 3690 struct mcx_cmd_set_issi_in *in; 3691 struct mcx_cmd_set_issi_out *out; 3692 uint8_t status; 3693 3694 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3695 3696 in = mcx_cmdq_in(cqe); 3697 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3698 in->cmd_op_mod = htobe16(0); 3699 in->cmd_current_issi = htobe16(MCX_ISSI); 3700 3701 mcx_cmdq_post(sc, cqe, slot); 3702 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3703 return (-1); 3704 if (mcx_cmdq_verify(cqe) != 0) 3705 return (-1); 3706 3707 status = cqe->cq_output_data[0]; 3708 if (status != MCX_CQ_STATUS_OK) 3709 return (-1); 3710 3711 return (0); 3712 } 3713 3714 static int 3715 mcx_issi(struct mcx_softc *sc) 3716 { 3717 struct mcx_dmamem mxm; 3718 struct mcx_cmdq_entry *cqe; 3719 struct mcx_cmd_query_issi_in *in; 3720 struct mcx_cmd_query_issi_il_out *out; 3721 struct mcx_cmd_query_issi_mb_out *mb; 3722 uint8_t token = mcx_cmdq_token(sc); 3723 uint8_t status; 3724 int error; 3725 3726 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3727 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3728 3729 in = mcx_cmdq_in(cqe); 3730 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3731 in->cmd_op_mod = htobe16(0); 3732 3733 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3734 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3735 &cqe->cq_output_ptr, token) != 0) { 3736 printf(", unable to allocate query issi mailbox\n"); 3737 return (-1); 3738 } 3739 mcx_cmdq_mboxes_sign(&mxm, 1); 3740 3741 mcx_cmdq_post(sc, cqe, 0); 3742 error = mcx_cmdq_poll(sc, cqe, 1000); 3743 if (error != 0) { 3744 printf(", query issi timeout\n"); 3745 goto free; 3746 } 3747 error = mcx_cmdq_verify(cqe); 3748 if (error != 0) { 3749 printf(", query issi reply corrupt\n"); 3750 goto free; 3751 } 3752 3753 status = cqe->cq_output_data[0]; 3754 switch (status) { 3755 case MCX_CQ_STATUS_OK: 3756 break; 3757 case MCX_CQ_STATUS_BAD_OPCODE: 3758 /* use ISSI 0 */ 3759 goto free; 3760 default: 3761 printf(", query issi failed (%x)\n", status); 3762 error = -1; 3763 goto free; 3764 } 3765 3766 out = mcx_cmdq_out(cqe); 3767 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3768 /* use ISSI 1 */ 3769 goto free; 3770 } 3771 3772 /* don't need to read cqe anymore, can be used for SET ISSI */ 3773 3774 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3775 CTASSERT(MCX_ISSI < NBBY); 3776 /* XXX math is hard */ 3777 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3778 /* use ISSI 0 */ 3779 goto free; 3780 } 3781 3782 if (mcx_set_issi(sc, cqe, 0) != 0) { 3783 /* ignore the error, just use ISSI 0 */ 3784 } else { 3785 /* use ISSI 1 */ 3786 } 3787 3788 free: 3789 mcx_cq_mboxes_free(sc, &mxm); 3790 return (error); 3791 } 3792 3793 static int 3794 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3795 int32_t *npages, uint16_t *func_id) 3796 { 3797 struct mcx_cmdq_entry *cqe; 3798 struct mcx_cmd_query_pages_in *in; 3799 struct mcx_cmd_query_pages_out *out; 3800 3801 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3802 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3803 3804 in = mcx_cmdq_in(cqe); 3805 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3806 in->cmd_op_mod = type; 3807 3808 mcx_cmdq_post(sc, cqe, 0); 3809 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3810 printf(", query pages timeout\n"); 3811 return (-1); 3812 } 3813 if (mcx_cmdq_verify(cqe) != 0) { 3814 printf(", query pages reply corrupt\n"); 3815 return (-1); 3816 } 3817 3818 out = mcx_cmdq_out(cqe); 3819 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3820 printf(", query pages failed (%x)\n", out->cmd_status); 3821 return (-1); 3822 } 3823 3824 *func_id = out->cmd_func_id; 3825 *npages = be32dec(&out->cmd_num_pages); 3826 3827 return (0); 3828 } 3829 3830 struct bus_dma_iter { 3831 bus_dmamap_t i_map; 3832 bus_size_t i_offset; 3833 unsigned int i_index; 3834 }; 3835 3836 static void 3837 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3838 { 3839 i->i_map = map; 3840 i->i_offset = 0; 3841 i->i_index = 0; 3842 } 3843 3844 static bus_addr_t 3845 bus_dma_iter_addr(struct bus_dma_iter *i) 3846 { 3847 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3848 } 3849 3850 static void 3851 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3852 { 3853 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3854 bus_size_t diff; 3855 3856 do { 3857 diff = seg->ds_len - i->i_offset; 3858 if (size < diff) 3859 break; 3860 3861 size -= diff; 3862 3863 seg++; 3864 3865 i->i_offset = 0; 3866 i->i_index++; 3867 } while (size > 0); 3868 3869 i->i_offset += size; 3870 } 3871 3872 static int 3873 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3874 { 3875 struct mcx_dmamem mxm; 3876 struct mcx_cmdq_entry *cqe; 3877 struct mcx_cmd_manage_pages_in *in; 3878 struct mcx_cmd_manage_pages_out *out; 3879 unsigned int paslen, nmb, i, j, npages; 3880 struct bus_dma_iter iter; 3881 uint64_t *pas; 3882 uint8_t status; 3883 uint8_t token = mcx_cmdq_token(sc); 3884 int error; 3885 3886 npages = mhm->mhm_npages; 3887 3888 paslen = sizeof(*pas) * npages; 3889 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3890 3891 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3892 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3893 3894 in = mcx_cmdq_in(cqe); 3895 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3896 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3897 in->cmd_func_id = func_id; 3898 be32enc(&in->cmd_input_num_entries, npages); 3899 3900 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3901 &cqe->cq_input_ptr, token) != 0) { 3902 printf(", unable to allocate manage pages mailboxen\n"); 3903 return (-1); 3904 } 3905 3906 bus_dma_iter_init(&iter, mhm->mhm_map); 3907 for (i = 0; i < nmb; i++) { 3908 unsigned int lim; 3909 3910 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3911 lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3912 3913 for (j = 0; j < lim; j++) { 3914 be64enc(&pas[j], bus_dma_iter_addr(&iter)); 3915 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3916 } 3917 3918 npages -= lim; 3919 } 3920 3921 mcx_cmdq_mboxes_sign(&mxm, nmb); 3922 3923 mcx_cmdq_post(sc, cqe, 0); 3924 error = mcx_cmdq_poll(sc, cqe, 1000); 3925 if (error != 0) { 3926 printf(", manage pages timeout\n"); 3927 goto free; 3928 } 3929 error = mcx_cmdq_verify(cqe); 3930 if (error != 0) { 3931 printf(", manage pages reply corrupt\n"); 3932 goto free; 3933 } 3934 3935 status = cqe->cq_output_data[0]; 3936 if (status != MCX_CQ_STATUS_OK) { 3937 printf(", manage pages failed (%x)\n", status); 3938 error = -1; 3939 goto free; 3940 } 3941 3942 free: 3943 mcx_dmamem_free(sc, &mxm); 3944 3945 return (error); 3946 } 3947 3948 static int 3949 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3950 { 3951 int32_t npages; 3952 uint16_t func_id; 3953 3954 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3955 /* error printed by mcx_query_pages */ 3956 return (-1); 3957 } 3958 3959 if (npages < 1) 3960 return (0); 3961 3962 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3963 printf(", unable to allocate hwmem\n"); 3964 return (-1); 3965 } 3966 3967 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3968 printf(", unable to add hwmem\n"); 3969 goto free; 3970 } 3971 3972 return (0); 3973 3974 free: 3975 mcx_hwmem_free(sc, mhm); 3976 3977 return (-1); 3978 } 3979 3980 static int 3981 mcx_hca_max_caps(struct mcx_softc *sc) 3982 { 3983 struct mcx_dmamem mxm; 3984 struct mcx_cmdq_entry *cqe; 3985 struct mcx_cmd_query_hca_cap_in *in; 3986 struct mcx_cmd_query_hca_cap_out *out; 3987 struct mcx_cmdq_mailbox *mb; 3988 struct mcx_cap_device *hca; 3989 uint8_t status; 3990 uint8_t token = mcx_cmdq_token(sc); 3991 int error; 3992 3993 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3994 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3995 token); 3996 3997 in = mcx_cmdq_in(cqe); 3998 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3999 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 4000 MCX_CMD_QUERY_HCA_CAP_DEVICE); 4001 4002 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 4003 &cqe->cq_output_ptr, token) != 0) { 4004 printf(", unable to allocate query hca caps mailboxen\n"); 4005 return (-1); 4006 } 4007 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 4008 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 4009 4010 mcx_cmdq_post(sc, cqe, 0); 4011 error = mcx_cmdq_poll(sc, cqe, 1000); 4012 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 4013 4014 if (error != 0) { 4015 printf(", query hca caps timeout\n"); 4016 goto free; 4017 } 4018 error = mcx_cmdq_verify(cqe); 4019 if (error != 0) { 4020 printf(", query hca caps reply corrupt\n"); 4021 goto free; 4022 } 4023 4024 status = cqe->cq_output_data[0]; 4025 if (status != MCX_CQ_STATUS_OK) { 4026 printf(", query hca caps failed (%x)\n", status); 4027 error = -1; 4028 goto free; 4029 } 4030 4031 mb = mcx_cq_mbox(&mxm, 0); 4032 hca = mcx_cq_mbox_data(mb); 4033 4034 if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE) 4035 != MCX_CAP_DEVICE_PORT_TYPE_ETH) { 4036 printf(", not in ethernet mode\n"); 4037 error = -1; 4038 goto free; 4039 } 4040 if (hca->log_pg_sz > PAGE_SHIFT) { 4041 printf(", minimum system page shift %u is too large\n", 4042 hca->log_pg_sz); 4043 error = -1; 4044 goto free; 4045 } 4046 /* 4047 * blueflame register is split into two buffers, and we must alternate 4048 * between the two of them. 4049 */ 4050 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 4051 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 4052 4053 if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG) 4054 sc->sc_mcam_reg = 1; 4055 4056 sc->sc_mhz = be32dec(&hca->device_frequency_mhz); 4057 sc->sc_khz = be32dec(&hca->device_frequency_khz); 4058 4059 free: 4060 mcx_dmamem_free(sc, &mxm); 4061 4062 return (error); 4063 } 4064 4065 static int 4066 mcx_hca_set_caps(struct mcx_softc *sc) 4067 { 4068 struct mcx_dmamem mxm; 4069 struct mcx_cmdq_entry *cqe; 4070 struct mcx_cmd_query_hca_cap_in *in; 4071 struct mcx_cmd_query_hca_cap_out *out; 4072 struct mcx_cmdq_mailbox *mb; 4073 struct mcx_cap_device *hca; 4074 uint8_t status; 4075 uint8_t token = mcx_cmdq_token(sc); 4076 int error; 4077 4078 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4079 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 4080 token); 4081 4082 in = mcx_cmdq_in(cqe); 4083 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 4084 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 4085 MCX_CMD_QUERY_HCA_CAP_DEVICE); 4086 4087 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 4088 &cqe->cq_output_ptr, token) != 0) { 4089 printf(", unable to allocate manage pages mailboxen\n"); 4090 return (-1); 4091 } 4092 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 4093 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 4094 4095 mcx_cmdq_post(sc, cqe, 0); 4096 error = mcx_cmdq_poll(sc, cqe, 1000); 4097 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 4098 4099 if (error != 0) { 4100 printf(", query hca caps timeout\n"); 4101 goto free; 4102 } 4103 error = mcx_cmdq_verify(cqe); 4104 if (error != 0) { 4105 printf(", query hca caps reply corrupt\n"); 4106 goto free; 4107 } 4108 4109 status = cqe->cq_output_data[0]; 4110 if (status != MCX_CQ_STATUS_OK) { 4111 printf(", query hca caps failed (%x)\n", status); 4112 error = -1; 4113 goto free; 4114 } 4115 4116 mb = mcx_cq_mbox(&mxm, 0); 4117 hca = mcx_cq_mbox_data(mb); 4118 4119 hca->log_pg_sz = PAGE_SHIFT; 4120 4121 free: 4122 mcx_dmamem_free(sc, &mxm); 4123 4124 return (error); 4125 } 4126 4127 4128 static int 4129 mcx_init_hca(struct mcx_softc *sc) 4130 { 4131 struct mcx_cmdq_entry *cqe; 4132 struct mcx_cmd_init_hca_in *in; 4133 struct mcx_cmd_init_hca_out *out; 4134 int error; 4135 uint8_t status; 4136 4137 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4138 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4139 4140 in = mcx_cmdq_in(cqe); 4141 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 4142 in->cmd_op_mod = htobe16(0); 4143 4144 mcx_cmdq_post(sc, cqe, 0); 4145 4146 error = mcx_cmdq_poll(sc, cqe, 1000); 4147 if (error != 0) { 4148 printf(", hca init timeout\n"); 4149 return (-1); 4150 } 4151 if (mcx_cmdq_verify(cqe) != 0) { 4152 printf(", hca init command corrupt\n"); 4153 return (-1); 4154 } 4155 4156 status = cqe->cq_output_data[0]; 4157 if (status != MCX_CQ_STATUS_OK) { 4158 printf(", hca init failed (%x)\n", status); 4159 return (-1); 4160 } 4161 4162 return (0); 4163 } 4164 4165 static int 4166 mcx_set_driver_version(struct mcx_softc *sc) 4167 { 4168 struct mcx_dmamem mxm; 4169 struct mcx_cmdq_entry *cqe; 4170 struct mcx_cmd_set_driver_version_in *in; 4171 struct mcx_cmd_set_driver_version_out *out; 4172 int error; 4173 int token; 4174 uint8_t status; 4175 4176 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4177 token = mcx_cmdq_token(sc); 4178 mcx_cmdq_init(sc, cqe, sizeof(*in) + 4179 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 4180 4181 in = mcx_cmdq_in(cqe); 4182 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 4183 in->cmd_op_mod = htobe16(0); 4184 4185 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4186 &cqe->cq_input_ptr, token) != 0) { 4187 printf(", unable to allocate set driver version mailboxen\n"); 4188 return (-1); 4189 } 4190 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 4191 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 4192 4193 mcx_cmdq_mboxes_sign(&mxm, 1); 4194 mcx_cmdq_post(sc, cqe, 0); 4195 4196 error = mcx_cmdq_poll(sc, cqe, 1000); 4197 if (error != 0) { 4198 printf(", set driver version timeout\n"); 4199 goto free; 4200 } 4201 if (mcx_cmdq_verify(cqe) != 0) { 4202 printf(", set driver version command corrupt\n"); 4203 goto free; 4204 } 4205 4206 status = cqe->cq_output_data[0]; 4207 if (status != MCX_CQ_STATUS_OK) { 4208 printf(", set driver version failed (%x)\n", status); 4209 error = -1; 4210 goto free; 4211 } 4212 4213 free: 4214 mcx_dmamem_free(sc, &mxm); 4215 4216 return (error); 4217 } 4218 4219 static int 4220 mcx_iff(struct mcx_softc *sc) 4221 { 4222 struct ifnet *ifp = &sc->sc_ec.ec_if; 4223 struct mcx_dmamem mxm; 4224 struct mcx_cmdq_entry *cqe; 4225 struct mcx_cmd_modify_nic_vport_context_in *in; 4226 struct mcx_cmd_modify_nic_vport_context_out *out; 4227 struct mcx_nic_vport_ctx *ctx; 4228 int error; 4229 int token; 4230 int insize; 4231 uint32_t dest; 4232 4233 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 4234 sc->sc_rss_flow_table_id; 4235 4236 /* enable or disable the promisc flow */ 4237 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 4238 if (sc->sc_promisc_flow_enabled == 0) { 4239 mcx_set_flow_table_entry_mac(sc, 4240 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 4241 sc->sc_promisc_flow_enabled = 1; 4242 } 4243 } else if (sc->sc_promisc_flow_enabled != 0) { 4244 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 4245 sc->sc_promisc_flow_enabled = 0; 4246 } 4247 4248 /* enable or disable the all-multicast flow */ 4249 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 4250 if (sc->sc_allmulti_flow_enabled == 0) { 4251 uint8_t mcast[ETHER_ADDR_LEN]; 4252 4253 memset(mcast, 0, sizeof(mcast)); 4254 mcast[0] = 0x01; 4255 mcx_set_flow_table_entry_mac(sc, 4256 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 4257 sc->sc_allmulti_flow_enabled = 1; 4258 } 4259 } else if (sc->sc_allmulti_flow_enabled != 0) { 4260 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 4261 sc->sc_allmulti_flow_enabled = 0; 4262 } 4263 4264 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 4265 4266 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4267 token = mcx_cmdq_token(sc); 4268 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4269 4270 in = mcx_cmdq_in(cqe); 4271 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 4272 in->cmd_op_mod = htobe16(0); 4273 in->cmd_field_select = htobe32( 4274 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 4275 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 4276 4277 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4278 printf(", unable to allocate modify " 4279 "nic vport context mailboxen\n"); 4280 return (-1); 4281 } 4282 ctx = (struct mcx_nic_vport_ctx *) 4283 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 4284 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 4285 /* 4286 * always leave promisc-all enabled on the vport since we 4287 * can't give it a vlan list, and we're already doing multicast 4288 * filtering in the flow table. 4289 */ 4290 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 4291 4292 mcx_cmdq_mboxes_sign(&mxm, 1); 4293 mcx_cmdq_post(sc, cqe, 0); 4294 4295 error = mcx_cmdq_poll(sc, cqe, 1000); 4296 if (error != 0) { 4297 printf(", modify nic vport context timeout\n"); 4298 goto free; 4299 } 4300 if (mcx_cmdq_verify(cqe) != 0) { 4301 printf(", modify nic vport context command corrupt\n"); 4302 goto free; 4303 } 4304 4305 out = mcx_cmdq_out(cqe); 4306 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4307 printf(", modify nic vport context failed (%x, %x)\n", 4308 out->cmd_status, be32toh(out->cmd_syndrome)); 4309 error = -1; 4310 goto free; 4311 } 4312 4313 free: 4314 mcx_dmamem_free(sc, &mxm); 4315 4316 return (error); 4317 } 4318 4319 static int 4320 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 4321 { 4322 struct mcx_cmdq_entry *cqe; 4323 struct mcx_cmd_alloc_uar_in *in; 4324 struct mcx_cmd_alloc_uar_out *out; 4325 int error; 4326 4327 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4328 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4329 4330 in = mcx_cmdq_in(cqe); 4331 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 4332 in->cmd_op_mod = htobe16(0); 4333 4334 mcx_cmdq_post(sc, cqe, 0); 4335 4336 error = mcx_cmdq_poll(sc, cqe, 1000); 4337 if (error != 0) { 4338 printf(", alloc uar timeout\n"); 4339 return (-1); 4340 } 4341 if (mcx_cmdq_verify(cqe) != 0) { 4342 printf(", alloc uar command corrupt\n"); 4343 return (-1); 4344 } 4345 4346 out = mcx_cmdq_out(cqe); 4347 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4348 printf(", alloc uar failed (%x)\n", out->cmd_status); 4349 return (-1); 4350 } 4351 4352 *uar = mcx_get_id(out->cmd_uar); 4353 return (0); 4354 } 4355 4356 static int 4357 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4358 uint64_t events, int vector) 4359 { 4360 struct mcx_cmdq_entry *cqe; 4361 struct mcx_dmamem mxm; 4362 struct mcx_cmd_create_eq_in *in; 4363 struct mcx_cmd_create_eq_mb_in *mbin; 4364 struct mcx_cmd_create_eq_out *out; 4365 struct mcx_eq_entry *eqe; 4366 int error; 4367 uint64_t *pas; 4368 int insize, npages, paslen, i, token; 4369 4370 eq->eq_cons = 0; 4371 4372 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4373 MCX_PAGE_SIZE); 4374 paslen = npages * sizeof(*pas); 4375 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4376 4377 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4378 MCX_PAGE_SIZE) != 0) { 4379 printf(", unable to allocate event queue memory\n"); 4380 return (-1); 4381 } 4382 4383 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4384 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4385 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4386 } 4387 4388 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4389 token = mcx_cmdq_token(sc); 4390 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4391 4392 in = mcx_cmdq_in(cqe); 4393 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4394 in->cmd_op_mod = htobe16(0); 4395 4396 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4397 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4398 &cqe->cq_input_ptr, token) != 0) { 4399 printf(", unable to allocate create eq mailboxen\n"); 4400 goto free_eq; 4401 } 4402 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4403 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4404 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4405 mbin->cmd_eq_ctx.eq_intr = vector; 4406 mbin->cmd_event_bitmask = htobe64(events); 4407 4408 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4409 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 4410 4411 /* physical addresses follow the mailbox in data */ 4412 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4413 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4414 mcx_cmdq_post(sc, cqe, 0); 4415 4416 error = mcx_cmdq_poll(sc, cqe, 1000); 4417 if (error != 0) { 4418 printf(", create eq timeout\n"); 4419 goto free_mxm; 4420 } 4421 if (mcx_cmdq_verify(cqe) != 0) { 4422 printf(", create eq command corrupt\n"); 4423 goto free_mxm; 4424 } 4425 4426 out = mcx_cmdq_out(cqe); 4427 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4428 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4429 be32toh(out->cmd_syndrome)); 4430 goto free_mxm; 4431 } 4432 4433 eq->eq_n = mcx_get_id(out->cmd_eqn); 4434 4435 mcx_dmamem_free(sc, &mxm); 4436 4437 mcx_arm_eq(sc, eq, uar); 4438 4439 return (0); 4440 4441 free_mxm: 4442 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 4443 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 4444 mcx_dmamem_free(sc, &mxm); 4445 free_eq: 4446 mcx_dmamem_free(sc, &eq->eq_mem); 4447 return (-1); 4448 } 4449 4450 static int 4451 mcx_alloc_pd(struct mcx_softc *sc) 4452 { 4453 struct mcx_cmdq_entry *cqe; 4454 struct mcx_cmd_alloc_pd_in *in; 4455 struct mcx_cmd_alloc_pd_out *out; 4456 int error; 4457 4458 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4459 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4460 4461 in = mcx_cmdq_in(cqe); 4462 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4463 in->cmd_op_mod = htobe16(0); 4464 4465 mcx_cmdq_post(sc, cqe, 0); 4466 4467 error = mcx_cmdq_poll(sc, cqe, 1000); 4468 if (error != 0) { 4469 printf(", alloc pd timeout\n"); 4470 return (-1); 4471 } 4472 if (mcx_cmdq_verify(cqe) != 0) { 4473 printf(", alloc pd command corrupt\n"); 4474 return (-1); 4475 } 4476 4477 out = mcx_cmdq_out(cqe); 4478 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4479 printf(", alloc pd failed (%x)\n", out->cmd_status); 4480 return (-1); 4481 } 4482 4483 sc->sc_pd = mcx_get_id(out->cmd_pd); 4484 return (0); 4485 } 4486 4487 static int 4488 mcx_alloc_tdomain(struct mcx_softc *sc) 4489 { 4490 struct mcx_cmdq_entry *cqe; 4491 struct mcx_cmd_alloc_td_in *in; 4492 struct mcx_cmd_alloc_td_out *out; 4493 int error; 4494 4495 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4496 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4497 4498 in = mcx_cmdq_in(cqe); 4499 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4500 in->cmd_op_mod = htobe16(0); 4501 4502 mcx_cmdq_post(sc, cqe, 0); 4503 4504 error = mcx_cmdq_poll(sc, cqe, 1000); 4505 if (error != 0) { 4506 printf(", alloc transport domain timeout\n"); 4507 return (-1); 4508 } 4509 if (mcx_cmdq_verify(cqe) != 0) { 4510 printf(", alloc transport domain command corrupt\n"); 4511 return (-1); 4512 } 4513 4514 out = mcx_cmdq_out(cqe); 4515 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4516 printf(", alloc transport domain failed (%x)\n", 4517 out->cmd_status); 4518 return (-1); 4519 } 4520 4521 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4522 return (0); 4523 } 4524 4525 static int 4526 mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr) 4527 { 4528 struct mcx_dmamem mxm; 4529 struct mcx_cmdq_entry *cqe; 4530 struct mcx_cmd_query_nic_vport_context_in *in; 4531 struct mcx_cmd_query_nic_vport_context_out *out; 4532 struct mcx_nic_vport_ctx *ctx; 4533 uint8_t *addr; 4534 int error, token, i; 4535 4536 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4537 token = mcx_cmdq_token(sc); 4538 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4539 4540 in = mcx_cmdq_in(cqe); 4541 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4542 in->cmd_op_mod = htobe16(0); 4543 in->cmd_allowed_list_type = 0; 4544 4545 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4546 &cqe->cq_output_ptr, token) != 0) { 4547 printf(", unable to allocate " 4548 "query nic vport context mailboxen\n"); 4549 return (-1); 4550 } 4551 mcx_cmdq_mboxes_sign(&mxm, 1); 4552 mcx_cmdq_post(sc, cqe, 0); 4553 4554 error = mcx_cmdq_poll(sc, cqe, 1000); 4555 if (error != 0) { 4556 printf(", query nic vport context timeout\n"); 4557 goto free; 4558 } 4559 if (mcx_cmdq_verify(cqe) != 0) { 4560 printf(", query nic vport context command corrupt\n"); 4561 goto free; 4562 } 4563 4564 out = mcx_cmdq_out(cqe); 4565 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4566 printf(", query nic vport context failed (%x, %x)\n", 4567 out->cmd_status, be32toh(out->cmd_syndrome)); 4568 error = -1; 4569 goto free; 4570 } 4571 4572 ctx = (struct mcx_nic_vport_ctx *) 4573 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4574 addr = (uint8_t *)&ctx->vp_perm_addr; 4575 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4576 enaddr[i] = addr[i + 2]; 4577 } 4578 free: 4579 mcx_dmamem_free(sc, &mxm); 4580 4581 return (error); 4582 } 4583 4584 static int 4585 mcx_query_special_contexts(struct mcx_softc *sc) 4586 { 4587 struct mcx_cmdq_entry *cqe; 4588 struct mcx_cmd_query_special_ctx_in *in; 4589 struct mcx_cmd_query_special_ctx_out *out; 4590 int error; 4591 4592 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4593 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4594 4595 in = mcx_cmdq_in(cqe); 4596 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4597 in->cmd_op_mod = htobe16(0); 4598 4599 mcx_cmdq_post(sc, cqe, 0); 4600 4601 error = mcx_cmdq_poll(sc, cqe, 1000); 4602 if (error != 0) { 4603 printf(", query special contexts timeout\n"); 4604 return (-1); 4605 } 4606 if (mcx_cmdq_verify(cqe) != 0) { 4607 printf(", query special contexts command corrupt\n"); 4608 return (-1); 4609 } 4610 4611 out = mcx_cmdq_out(cqe); 4612 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4613 printf(", query special contexts failed (%x)\n", 4614 out->cmd_status); 4615 return (-1); 4616 } 4617 4618 sc->sc_lkey = be32toh(out->cmd_resd_lkey); 4619 return (0); 4620 } 4621 4622 static int 4623 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4624 { 4625 struct mcx_reg_pmtu pmtu; 4626 int error; 4627 4628 /* read max mtu */ 4629 memset(&pmtu, 0, sizeof(pmtu)); 4630 pmtu.rp_local_port = 1; 4631 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4632 sizeof(pmtu)); 4633 if (error != 0) { 4634 printf(", unable to get port MTU\n"); 4635 return error; 4636 } 4637 4638 mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu)); 4639 pmtu.rp_admin_mtu = htobe16(mtu); 4640 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4641 sizeof(pmtu)); 4642 if (error != 0) { 4643 printf(", unable to set port MTU\n"); 4644 return error; 4645 } 4646 4647 sc->sc_hardmtu = mtu; 4648 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4649 return 0; 4650 } 4651 4652 static int 4653 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4654 { 4655 struct mcx_cmdq_entry *cmde; 4656 struct mcx_cq_entry *cqe; 4657 struct mcx_dmamem mxm; 4658 struct mcx_cmd_create_cq_in *in; 4659 struct mcx_cmd_create_cq_mb_in *mbin; 4660 struct mcx_cmd_create_cq_out *out; 4661 int error; 4662 uint64_t *pas; 4663 int insize, npages, paslen, i, token; 4664 4665 cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4666 4667 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4668 MCX_PAGE_SIZE); 4669 paslen = npages * sizeof(*pas); 4670 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4671 4672 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4673 MCX_PAGE_SIZE) != 0) { 4674 printf("%s: unable to allocate completion queue memory\n", 4675 DEVNAME(sc)); 4676 return (-1); 4677 } 4678 cqe = MCX_DMA_KVA(&cq->cq_mem); 4679 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4680 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4681 } 4682 4683 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4684 token = mcx_cmdq_token(sc); 4685 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4686 4687 in = mcx_cmdq_in(cmde); 4688 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4689 in->cmd_op_mod = htobe16(0); 4690 4691 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4692 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4693 &cmde->cq_input_ptr, token) != 0) { 4694 printf("%s: unable to allocate create cq mailboxen\n", 4695 DEVNAME(sc)); 4696 goto free_cq; 4697 } 4698 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4699 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4700 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4701 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4702 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4703 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4704 MCX_CQ_MOD_COUNTER); 4705 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4706 MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell); 4707 4708 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4709 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 4710 4711 /* physical addresses follow the mailbox in data */ 4712 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4713 mcx_cmdq_post(sc, cmde, 0); 4714 4715 error = mcx_cmdq_poll(sc, cmde, 1000); 4716 if (error != 0) { 4717 printf("%s: create cq timeout\n", DEVNAME(sc)); 4718 goto free_mxm; 4719 } 4720 if (mcx_cmdq_verify(cmde) != 0) { 4721 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4722 goto free_mxm; 4723 } 4724 4725 out = mcx_cmdq_out(cmde); 4726 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4727 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4728 out->cmd_status, be32toh(out->cmd_syndrome)); 4729 goto free_mxm; 4730 } 4731 4732 cq->cq_n = mcx_get_id(out->cmd_cqn); 4733 cq->cq_cons = 0; 4734 cq->cq_count = 0; 4735 4736 mcx_dmamem_free(sc, &mxm); 4737 4738 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4739 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4740 BUS_DMASYNC_PREWRITE); 4741 4742 mcx_arm_cq(sc, cq, uar); 4743 4744 return (0); 4745 4746 free_mxm: 4747 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4748 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4749 mcx_dmamem_free(sc, &mxm); 4750 free_cq: 4751 mcx_dmamem_free(sc, &cq->cq_mem); 4752 return (-1); 4753 } 4754 4755 static int 4756 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4757 { 4758 struct mcx_cmdq_entry *cqe; 4759 struct mcx_cmd_destroy_cq_in *in; 4760 struct mcx_cmd_destroy_cq_out *out; 4761 int error; 4762 int token; 4763 4764 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4765 token = mcx_cmdq_token(sc); 4766 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4767 4768 in = mcx_cmdq_in(cqe); 4769 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4770 in->cmd_op_mod = htobe16(0); 4771 in->cmd_cqn = htobe32(cq->cq_n); 4772 4773 mcx_cmdq_post(sc, cqe, 0); 4774 error = mcx_cmdq_poll(sc, cqe, 1000); 4775 if (error != 0) { 4776 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4777 return error; 4778 } 4779 if (mcx_cmdq_verify(cqe) != 0) { 4780 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4781 return error; 4782 } 4783 4784 out = mcx_cmdq_out(cqe); 4785 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4786 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4787 out->cmd_status, be32toh(out->cmd_syndrome)); 4788 return -1; 4789 } 4790 4791 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4792 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell), 4793 BUS_DMASYNC_POSTWRITE); 4794 4795 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 4796 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 4797 mcx_dmamem_free(sc, &cq->cq_mem); 4798 4799 cq->cq_n = 0; 4800 cq->cq_cons = 0; 4801 cq->cq_count = 0; 4802 return 0; 4803 } 4804 4805 static int 4806 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4807 { 4808 struct mcx_cmdq_entry *cqe; 4809 struct mcx_dmamem mxm; 4810 struct mcx_cmd_create_rq_in *in; 4811 struct mcx_cmd_create_rq_out *out; 4812 struct mcx_rq_ctx *mbin; 4813 int error; 4814 uint64_t *pas; 4815 uint32_t rq_flags; 4816 int insize, npages, paslen, token; 4817 4818 rx->rx_doorbell = MCX_WQ_DOORBELL_BASE + 4819 (db * MCX_WQ_DOORBELL_STRIDE); 4820 4821 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4822 MCX_PAGE_SIZE); 4823 paslen = npages * sizeof(*pas); 4824 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4825 4826 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4827 MCX_PAGE_SIZE) != 0) { 4828 printf("%s: unable to allocate receive queue memory\n", 4829 DEVNAME(sc)); 4830 return (-1); 4831 } 4832 4833 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4834 token = mcx_cmdq_token(sc); 4835 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4836 4837 in = mcx_cmdq_in(cqe); 4838 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4839 in->cmd_op_mod = htobe16(0); 4840 4841 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4842 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4843 &cqe->cq_input_ptr, token) != 0) { 4844 printf("%s: unable to allocate create rq mailboxen\n", 4845 DEVNAME(sc)); 4846 goto free_rq; 4847 } 4848 mbin = (struct mcx_rq_ctx *) 4849 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4850 rq_flags = MCX_RQ_CTX_RLKEY; 4851 mbin->rq_flags = htobe32(rq_flags); 4852 mbin->rq_cqn = htobe32(cqn); 4853 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4854 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4855 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4856 rx->rx_doorbell); 4857 mbin->rq_wq.wq_log_stride = htobe16(4); 4858 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4859 4860 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4861 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 4862 4863 /* physical addresses follow the mailbox in data */ 4864 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4865 mcx_cmdq_post(sc, cqe, 0); 4866 4867 error = mcx_cmdq_poll(sc, cqe, 1000); 4868 if (error != 0) { 4869 printf("%s: create rq timeout\n", DEVNAME(sc)); 4870 goto free_mxm; 4871 } 4872 if (mcx_cmdq_verify(cqe) != 0) { 4873 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4874 goto free_mxm; 4875 } 4876 4877 out = mcx_cmdq_out(cqe); 4878 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4879 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4880 out->cmd_status, be32toh(out->cmd_syndrome)); 4881 goto free_mxm; 4882 } 4883 4884 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4885 4886 mcx_dmamem_free(sc, &mxm); 4887 4888 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4889 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 4890 4891 return (0); 4892 4893 free_mxm: 4894 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4895 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4896 mcx_dmamem_free(sc, &mxm); 4897 free_rq: 4898 mcx_dmamem_free(sc, &rx->rx_rq_mem); 4899 return (-1); 4900 } 4901 4902 static int 4903 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4904 { 4905 struct mcx_cmdq_entry *cqe; 4906 struct mcx_dmamem mxm; 4907 struct mcx_cmd_modify_rq_in *in; 4908 struct mcx_cmd_modify_rq_mb_in *mbin; 4909 struct mcx_cmd_modify_rq_out *out; 4910 int error; 4911 int token; 4912 4913 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4914 token = mcx_cmdq_token(sc); 4915 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4916 sizeof(*out), token); 4917 4918 in = mcx_cmdq_in(cqe); 4919 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4920 in->cmd_op_mod = htobe16(0); 4921 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4922 4923 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4924 &cqe->cq_input_ptr, token) != 0) { 4925 printf("%s: unable to allocate modify rq mailbox\n", 4926 DEVNAME(sc)); 4927 return (-1); 4928 } 4929 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4930 mbin->cmd_rq_ctx.rq_flags = htobe32( 4931 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4932 4933 mcx_cmdq_mboxes_sign(&mxm, 1); 4934 mcx_cmdq_post(sc, cqe, 0); 4935 error = mcx_cmdq_poll(sc, cqe, 1000); 4936 if (error != 0) { 4937 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4938 goto free; 4939 } 4940 if (mcx_cmdq_verify(cqe) != 0) { 4941 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4942 goto free; 4943 } 4944 4945 out = mcx_cmdq_out(cqe); 4946 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4947 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4948 out->cmd_status, be32toh(out->cmd_syndrome)); 4949 error = -1; 4950 goto free; 4951 } 4952 4953 free: 4954 mcx_dmamem_free(sc, &mxm); 4955 return (error); 4956 } 4957 4958 static int 4959 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4960 { 4961 struct mcx_cmdq_entry *cqe; 4962 struct mcx_cmd_destroy_rq_in *in; 4963 struct mcx_cmd_destroy_rq_out *out; 4964 int error; 4965 int token; 4966 4967 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4968 token = mcx_cmdq_token(sc); 4969 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4970 4971 in = mcx_cmdq_in(cqe); 4972 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4973 in->cmd_op_mod = htobe16(0); 4974 in->cmd_rqn = htobe32(rx->rx_rqn); 4975 4976 mcx_cmdq_post(sc, cqe, 0); 4977 error = mcx_cmdq_poll(sc, cqe, 1000); 4978 if (error != 0) { 4979 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4980 return error; 4981 } 4982 if (mcx_cmdq_verify(cqe) != 0) { 4983 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4984 return error; 4985 } 4986 4987 out = mcx_cmdq_out(cqe); 4988 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4989 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4990 out->cmd_status, be32toh(out->cmd_syndrome)); 4991 return -1; 4992 } 4993 4994 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 4995 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 4996 4997 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 4998 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 4999 mcx_dmamem_free(sc, &rx->rx_rq_mem); 5000 5001 rx->rx_rqn = 0; 5002 return 0; 5003 } 5004 5005 static int 5006 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 5007 { 5008 struct mcx_cmdq_entry *cqe; 5009 struct mcx_dmamem mxm; 5010 struct mcx_cmd_create_tir_in *in; 5011 struct mcx_cmd_create_tir_mb_in *mbin; 5012 struct mcx_cmd_create_tir_out *out; 5013 int error; 5014 int token; 5015 5016 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5017 token = mcx_cmdq_token(sc); 5018 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5019 sizeof(*out), token); 5020 5021 in = mcx_cmdq_in(cqe); 5022 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 5023 in->cmd_op_mod = htobe16(0); 5024 5025 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5026 &cqe->cq_input_ptr, token) != 0) { 5027 printf("%s: unable to allocate create tir mailbox\n", 5028 DEVNAME(sc)); 5029 return (-1); 5030 } 5031 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5032 /* leave disp_type = 0, so packets get sent to the inline rqn */ 5033 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 5034 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5035 5036 mcx_cmdq_post(sc, cqe, 0); 5037 error = mcx_cmdq_poll(sc, cqe, 1000); 5038 if (error != 0) { 5039 printf("%s: create tir timeout\n", DEVNAME(sc)); 5040 goto free; 5041 } 5042 if (mcx_cmdq_verify(cqe) != 0) { 5043 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 5044 goto free; 5045 } 5046 5047 out = mcx_cmdq_out(cqe); 5048 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5049 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 5050 out->cmd_status, be32toh(out->cmd_syndrome)); 5051 error = -1; 5052 goto free; 5053 } 5054 5055 *tirn = mcx_get_id(out->cmd_tirn); 5056 free: 5057 mcx_dmamem_free(sc, &mxm); 5058 return (error); 5059 } 5060 5061 static int 5062 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 5063 int *tirn) 5064 { 5065 struct mcx_cmdq_entry *cqe; 5066 struct mcx_dmamem mxm; 5067 struct mcx_cmd_create_tir_in *in; 5068 struct mcx_cmd_create_tir_mb_in *mbin; 5069 struct mcx_cmd_create_tir_out *out; 5070 int error; 5071 int token; 5072 5073 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5074 token = mcx_cmdq_token(sc); 5075 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5076 sizeof(*out), token); 5077 5078 in = mcx_cmdq_in(cqe); 5079 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 5080 in->cmd_op_mod = htobe16(0); 5081 5082 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5083 &cqe->cq_input_ptr, token) != 0) { 5084 printf("%s: unable to allocate create tir mailbox\n", 5085 DEVNAME(sc)); 5086 return (-1); 5087 } 5088 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5089 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 5090 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 5091 mbin->cmd_indir_table = htobe32(rqtn); 5092 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 5093 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 5094 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 5095 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 5096 sizeof(mbin->cmd_rx_hash_key)); 5097 5098 mcx_cmdq_post(sc, cqe, 0); 5099 error = mcx_cmdq_poll(sc, cqe, 1000); 5100 if (error != 0) { 5101 printf("%s: create tir timeout\n", DEVNAME(sc)); 5102 goto free; 5103 } 5104 if (mcx_cmdq_verify(cqe) != 0) { 5105 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 5106 goto free; 5107 } 5108 5109 out = mcx_cmdq_out(cqe); 5110 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5111 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 5112 out->cmd_status, be32toh(out->cmd_syndrome)); 5113 error = -1; 5114 goto free; 5115 } 5116 5117 *tirn = mcx_get_id(out->cmd_tirn); 5118 free: 5119 mcx_dmamem_free(sc, &mxm); 5120 return (error); 5121 } 5122 5123 static int 5124 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 5125 { 5126 struct mcx_cmdq_entry *cqe; 5127 struct mcx_cmd_destroy_tir_in *in; 5128 struct mcx_cmd_destroy_tir_out *out; 5129 int error; 5130 int token; 5131 5132 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5133 token = mcx_cmdq_token(sc); 5134 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5135 5136 in = mcx_cmdq_in(cqe); 5137 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 5138 in->cmd_op_mod = htobe16(0); 5139 in->cmd_tirn = htobe32(tirn); 5140 5141 mcx_cmdq_post(sc, cqe, 0); 5142 error = mcx_cmdq_poll(sc, cqe, 1000); 5143 if (error != 0) { 5144 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 5145 return error; 5146 } 5147 if (mcx_cmdq_verify(cqe) != 0) { 5148 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 5149 return error; 5150 } 5151 5152 out = mcx_cmdq_out(cqe); 5153 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5154 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 5155 out->cmd_status, be32toh(out->cmd_syndrome)); 5156 return -1; 5157 } 5158 5159 return (0); 5160 } 5161 5162 static int 5163 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 5164 int cqn) 5165 { 5166 struct mcx_cmdq_entry *cqe; 5167 struct mcx_dmamem mxm; 5168 struct mcx_cmd_create_sq_in *in; 5169 struct mcx_sq_ctx *mbin; 5170 struct mcx_cmd_create_sq_out *out; 5171 int error; 5172 uint64_t *pas; 5173 int insize, npages, paslen, token; 5174 5175 tx->tx_doorbell = MCX_WQ_DOORBELL_BASE + 5176 (db * MCX_WQ_DOORBELL_STRIDE) + 4; 5177 5178 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 5179 MCX_PAGE_SIZE); 5180 paslen = npages * sizeof(*pas); 5181 insize = sizeof(struct mcx_sq_ctx) + paslen; 5182 5183 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 5184 MCX_PAGE_SIZE) != 0) { 5185 printf("%s: unable to allocate send queue memory\n", 5186 DEVNAME(sc)); 5187 return (-1); 5188 } 5189 5190 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5191 token = mcx_cmdq_token(sc); 5192 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 5193 token); 5194 5195 in = mcx_cmdq_in(cqe); 5196 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 5197 in->cmd_op_mod = htobe16(0); 5198 5199 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 5200 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 5201 &cqe->cq_input_ptr, token) != 0) { 5202 printf("%s: unable to allocate create sq mailboxen\n", 5203 DEVNAME(sc)); 5204 goto free_sq; 5205 } 5206 mbin = (struct mcx_sq_ctx *) 5207 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 5208 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 5209 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 5210 mbin->sq_cqn = htobe32(cqn); 5211 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 5212 mbin->sq_tis_num = htobe32(sc->sc_tis); 5213 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 5214 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 5215 mbin->sq_wq.wq_uar_page = htobe32(uar); 5216 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 5217 tx->tx_doorbell); 5218 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 5219 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 5220 5221 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5222 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 5223 5224 /* physical addresses follow the mailbox in data */ 5225 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 5226 npages, &tx->tx_sq_mem); 5227 mcx_cmdq_post(sc, cqe, 0); 5228 5229 error = mcx_cmdq_poll(sc, cqe, 1000); 5230 if (error != 0) { 5231 printf("%s: create sq timeout\n", DEVNAME(sc)); 5232 goto free_mxm; 5233 } 5234 if (mcx_cmdq_verify(cqe) != 0) { 5235 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 5236 goto free_mxm; 5237 } 5238 5239 out = mcx_cmdq_out(cqe); 5240 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5241 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 5242 out->cmd_status, be32toh(out->cmd_syndrome)); 5243 goto free_mxm; 5244 } 5245 5246 tx->tx_uar = uar; 5247 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 5248 5249 mcx_dmamem_free(sc, &mxm); 5250 5251 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5252 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 5253 5254 return (0); 5255 5256 free_mxm: 5257 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5258 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5259 mcx_dmamem_free(sc, &mxm); 5260 free_sq: 5261 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5262 return (-1); 5263 } 5264 5265 static int 5266 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5267 { 5268 struct mcx_cmdq_entry *cqe; 5269 struct mcx_cmd_destroy_sq_in *in; 5270 struct mcx_cmd_destroy_sq_out *out; 5271 int error; 5272 int token; 5273 5274 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5275 token = mcx_cmdq_token(sc); 5276 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5277 5278 in = mcx_cmdq_in(cqe); 5279 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 5280 in->cmd_op_mod = htobe16(0); 5281 in->cmd_sqn = htobe32(tx->tx_sqn); 5282 5283 mcx_cmdq_post(sc, cqe, 0); 5284 error = mcx_cmdq_poll(sc, cqe, 1000); 5285 if (error != 0) { 5286 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 5287 return error; 5288 } 5289 if (mcx_cmdq_verify(cqe) != 0) { 5290 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 5291 return error; 5292 } 5293 5294 out = mcx_cmdq_out(cqe); 5295 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5296 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 5297 out->cmd_status, be32toh(out->cmd_syndrome)); 5298 return -1; 5299 } 5300 5301 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 5302 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 5303 5304 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 5305 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 5306 mcx_dmamem_free(sc, &tx->tx_sq_mem); 5307 5308 tx->tx_sqn = 0; 5309 return 0; 5310 } 5311 5312 static int 5313 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 5314 { 5315 struct mcx_cmdq_entry *cqe; 5316 struct mcx_dmamem mxm; 5317 struct mcx_cmd_modify_sq_in *in; 5318 struct mcx_cmd_modify_sq_mb_in *mbin; 5319 struct mcx_cmd_modify_sq_out *out; 5320 int error; 5321 int token; 5322 5323 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5324 token = mcx_cmdq_token(sc); 5325 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5326 sizeof(*out), token); 5327 5328 in = mcx_cmdq_in(cqe); 5329 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 5330 in->cmd_op_mod = htobe16(0); 5331 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 5332 5333 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5334 &cqe->cq_input_ptr, token) != 0) { 5335 printf("%s: unable to allocate modify sq mailbox\n", 5336 DEVNAME(sc)); 5337 return (-1); 5338 } 5339 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5340 mbin->cmd_sq_ctx.sq_flags = htobe32( 5341 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 5342 5343 mcx_cmdq_mboxes_sign(&mxm, 1); 5344 mcx_cmdq_post(sc, cqe, 0); 5345 error = mcx_cmdq_poll(sc, cqe, 1000); 5346 if (error != 0) { 5347 printf("%s: modify sq timeout\n", DEVNAME(sc)); 5348 goto free; 5349 } 5350 if (mcx_cmdq_verify(cqe) != 0) { 5351 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 5352 goto free; 5353 } 5354 5355 out = mcx_cmdq_out(cqe); 5356 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5357 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 5358 out->cmd_status, be32toh(out->cmd_syndrome)); 5359 error = -1; 5360 goto free; 5361 } 5362 5363 free: 5364 mcx_dmamem_free(sc, &mxm); 5365 return (error); 5366 } 5367 5368 static int 5369 mcx_create_tis(struct mcx_softc *sc, int *tis) 5370 { 5371 struct mcx_cmdq_entry *cqe; 5372 struct mcx_dmamem mxm; 5373 struct mcx_cmd_create_tis_in *in; 5374 struct mcx_cmd_create_tis_mb_in *mbin; 5375 struct mcx_cmd_create_tis_out *out; 5376 int error; 5377 int token; 5378 5379 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5380 token = mcx_cmdq_token(sc); 5381 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5382 sizeof(*out), token); 5383 5384 in = mcx_cmdq_in(cqe); 5385 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 5386 in->cmd_op_mod = htobe16(0); 5387 5388 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5389 &cqe->cq_input_ptr, token) != 0) { 5390 printf("%s: unable to allocate create tis mailbox\n", 5391 DEVNAME(sc)); 5392 return (-1); 5393 } 5394 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5395 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5396 5397 mcx_cmdq_mboxes_sign(&mxm, 1); 5398 mcx_cmdq_post(sc, cqe, 0); 5399 error = mcx_cmdq_poll(sc, cqe, 1000); 5400 if (error != 0) { 5401 printf("%s: create tis timeout\n", DEVNAME(sc)); 5402 goto free; 5403 } 5404 if (mcx_cmdq_verify(cqe) != 0) { 5405 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5406 goto free; 5407 } 5408 5409 out = mcx_cmdq_out(cqe); 5410 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5411 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5412 out->cmd_status, be32toh(out->cmd_syndrome)); 5413 error = -1; 5414 goto free; 5415 } 5416 5417 *tis = mcx_get_id(out->cmd_tisn); 5418 free: 5419 mcx_dmamem_free(sc, &mxm); 5420 return (error); 5421 } 5422 5423 static int 5424 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5425 { 5426 struct mcx_cmdq_entry *cqe; 5427 struct mcx_cmd_destroy_tis_in *in; 5428 struct mcx_cmd_destroy_tis_out *out; 5429 int error; 5430 int token; 5431 5432 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5433 token = mcx_cmdq_token(sc); 5434 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5435 5436 in = mcx_cmdq_in(cqe); 5437 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5438 in->cmd_op_mod = htobe16(0); 5439 in->cmd_tisn = htobe32(tis); 5440 5441 mcx_cmdq_post(sc, cqe, 0); 5442 error = mcx_cmdq_poll(sc, cqe, 1000); 5443 if (error != 0) { 5444 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5445 return error; 5446 } 5447 if (mcx_cmdq_verify(cqe) != 0) { 5448 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5449 return error; 5450 } 5451 5452 out = mcx_cmdq_out(cqe); 5453 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5454 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5455 out->cmd_status, be32toh(out->cmd_syndrome)); 5456 return -1; 5457 } 5458 5459 return 0; 5460 } 5461 5462 static int 5463 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5464 { 5465 struct mcx_cmdq_entry *cqe; 5466 struct mcx_dmamem mxm; 5467 struct mcx_cmd_create_rqt_in *in; 5468 struct mcx_cmd_create_rqt_mb_in *mbin; 5469 struct mcx_cmd_create_rqt_out *out; 5470 struct mcx_rqt_ctx *rqt_ctx; 5471 int *rqtn; 5472 int error; 5473 int token; 5474 int i; 5475 5476 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5477 token = mcx_cmdq_token(sc); 5478 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5479 (size * sizeof(int)), sizeof(*out), token); 5480 5481 in = mcx_cmdq_in(cqe); 5482 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5483 in->cmd_op_mod = htobe16(0); 5484 5485 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5486 &cqe->cq_input_ptr, token) != 0) { 5487 printf("%s: unable to allocate create rqt mailbox\n", 5488 DEVNAME(sc)); 5489 return (-1); 5490 } 5491 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5492 rqt_ctx = &mbin->cmd_rqt; 5493 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5494 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5495 5496 /* rqt list follows the rqt context */ 5497 rqtn = (int *)(rqt_ctx + 1); 5498 for (i = 0; i < size; i++) { 5499 rqtn[i] = htobe32(rqns[i]); 5500 } 5501 5502 mcx_cmdq_mboxes_sign(&mxm, 1); 5503 mcx_cmdq_post(sc, cqe, 0); 5504 error = mcx_cmdq_poll(sc, cqe, 1000); 5505 if (error != 0) { 5506 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5507 goto free; 5508 } 5509 if (mcx_cmdq_verify(cqe) != 0) { 5510 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5511 goto free; 5512 } 5513 5514 out = mcx_cmdq_out(cqe); 5515 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5516 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5517 out->cmd_status, be32toh(out->cmd_syndrome)); 5518 error = -1; 5519 goto free; 5520 } 5521 5522 *rqt = mcx_get_id(out->cmd_rqtn); 5523 return (0); 5524 free: 5525 mcx_dmamem_free(sc, &mxm); 5526 return (error); 5527 } 5528 5529 static int 5530 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5531 { 5532 struct mcx_cmdq_entry *cqe; 5533 struct mcx_cmd_destroy_rqt_in *in; 5534 struct mcx_cmd_destroy_rqt_out *out; 5535 int error; 5536 int token; 5537 5538 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5539 token = mcx_cmdq_token(sc); 5540 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5541 5542 in = mcx_cmdq_in(cqe); 5543 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5544 in->cmd_op_mod = htobe16(0); 5545 in->cmd_rqtn = htobe32(rqt); 5546 5547 mcx_cmdq_post(sc, cqe, 0); 5548 error = mcx_cmdq_poll(sc, cqe, 1000); 5549 if (error != 0) { 5550 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5551 return error; 5552 } 5553 if (mcx_cmdq_verify(cqe) != 0) { 5554 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5555 return error; 5556 } 5557 5558 out = mcx_cmdq_out(cqe); 5559 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5560 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5561 out->cmd_status, be32toh(out->cmd_syndrome)); 5562 return -1; 5563 } 5564 5565 return 0; 5566 } 5567 5568 #if 0 5569 static int 5570 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5571 { 5572 struct mcx_cmdq_entry *cqe; 5573 struct mcx_cmd_alloc_flow_counter_in *in; 5574 struct mcx_cmd_alloc_flow_counter_out *out; 5575 int error; 5576 5577 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5578 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5579 5580 in = mcx_cmdq_in(cqe); 5581 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5582 in->cmd_op_mod = htobe16(0); 5583 5584 mcx_cmdq_post(sc, cqe, 0); 5585 5586 error = mcx_cmdq_poll(sc, cqe, 1000); 5587 if (error != 0) { 5588 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5589 return (-1); 5590 } 5591 if (mcx_cmdq_verify(cqe) != 0) { 5592 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5593 return (-1); 5594 } 5595 5596 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5597 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5598 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5599 out->cmd_status); 5600 return (-1); 5601 } 5602 5603 sc->sc_flow_counter_id[i] = be16toh(out->cmd_flow_counter_id); 5604 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5605 5606 return (0); 5607 } 5608 #endif 5609 5610 static int 5611 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5612 int *flow_table_id) 5613 { 5614 struct mcx_cmdq_entry *cqe; 5615 struct mcx_dmamem mxm; 5616 struct mcx_cmd_create_flow_table_in *in; 5617 struct mcx_cmd_create_flow_table_mb_in *mbin; 5618 struct mcx_cmd_create_flow_table_out *out; 5619 int error; 5620 int token; 5621 5622 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5623 token = mcx_cmdq_token(sc); 5624 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5625 sizeof(*out), token); 5626 5627 in = mcx_cmdq_in(cqe); 5628 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5629 in->cmd_op_mod = htobe16(0); 5630 5631 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5632 &cqe->cq_input_ptr, token) != 0) { 5633 printf("%s: unable to allocate create flow table mailbox\n", 5634 DEVNAME(sc)); 5635 return (-1); 5636 } 5637 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5638 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5639 mbin->cmd_ctx.ft_log_size = log_size; 5640 mbin->cmd_ctx.ft_level = level; 5641 5642 mcx_cmdq_mboxes_sign(&mxm, 1); 5643 mcx_cmdq_post(sc, cqe, 0); 5644 error = mcx_cmdq_poll(sc, cqe, 1000); 5645 if (error != 0) { 5646 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5647 goto free; 5648 } 5649 if (mcx_cmdq_verify(cqe) != 0) { 5650 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5651 goto free; 5652 } 5653 5654 out = mcx_cmdq_out(cqe); 5655 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5656 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5657 out->cmd_status, be32toh(out->cmd_syndrome)); 5658 error = -1; 5659 goto free; 5660 } 5661 5662 *flow_table_id = mcx_get_id(out->cmd_table_id); 5663 free: 5664 mcx_dmamem_free(sc, &mxm); 5665 return (error); 5666 } 5667 5668 static int 5669 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5670 { 5671 struct mcx_cmdq_entry *cqe; 5672 struct mcx_dmamem mxm; 5673 struct mcx_cmd_set_flow_table_root_in *in; 5674 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5675 struct mcx_cmd_set_flow_table_root_out *out; 5676 int error; 5677 int token; 5678 5679 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5680 token = mcx_cmdq_token(sc); 5681 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5682 sizeof(*out), token); 5683 5684 in = mcx_cmdq_in(cqe); 5685 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5686 in->cmd_op_mod = htobe16(0); 5687 5688 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5689 &cqe->cq_input_ptr, token) != 0) { 5690 printf("%s: unable to allocate set flow table root mailbox\n", 5691 DEVNAME(sc)); 5692 return (-1); 5693 } 5694 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5695 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5696 mbin->cmd_table_id = htobe32(flow_table_id); 5697 5698 mcx_cmdq_mboxes_sign(&mxm, 1); 5699 mcx_cmdq_post(sc, cqe, 0); 5700 error = mcx_cmdq_poll(sc, cqe, 1000); 5701 if (error != 0) { 5702 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5703 goto free; 5704 } 5705 if (mcx_cmdq_verify(cqe) != 0) { 5706 printf("%s: set flow table root command corrupt\n", 5707 DEVNAME(sc)); 5708 goto free; 5709 } 5710 5711 out = mcx_cmdq_out(cqe); 5712 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5713 printf("%s: set flow table root failed (%x, %x)\n", 5714 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome)); 5715 error = -1; 5716 goto free; 5717 } 5718 5719 free: 5720 mcx_dmamem_free(sc, &mxm); 5721 return (error); 5722 } 5723 5724 static int 5725 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5726 { 5727 struct mcx_cmdq_entry *cqe; 5728 struct mcx_dmamem mxm; 5729 struct mcx_cmd_destroy_flow_table_in *in; 5730 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5731 struct mcx_cmd_destroy_flow_table_out *out; 5732 int error; 5733 int token; 5734 5735 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5736 token = mcx_cmdq_token(sc); 5737 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5738 5739 in = mcx_cmdq_in(cqe); 5740 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5741 in->cmd_op_mod = htobe16(0); 5742 5743 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5744 &cqe->cq_input_ptr, token) != 0) { 5745 printf("%s: unable to allocate destroy flow table mailbox\n", 5746 DEVNAME(sc)); 5747 return (-1); 5748 } 5749 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5750 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5751 mb->cmd_table_id = htobe32(flow_table_id); 5752 5753 mcx_cmdq_mboxes_sign(&mxm, 1); 5754 mcx_cmdq_post(sc, cqe, 0); 5755 error = mcx_cmdq_poll(sc, cqe, 1000); 5756 if (error != 0) { 5757 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5758 goto free; 5759 } 5760 if (mcx_cmdq_verify(cqe) != 0) { 5761 printf("%s: destroy flow table command corrupt\n", 5762 DEVNAME(sc)); 5763 goto free; 5764 } 5765 5766 out = mcx_cmdq_out(cqe); 5767 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5768 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5769 out->cmd_status, be32toh(out->cmd_syndrome)); 5770 error = -1; 5771 goto free; 5772 } 5773 5774 free: 5775 mcx_dmamem_free(sc, &mxm); 5776 return (error); 5777 } 5778 5779 5780 static int 5781 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5782 int start, int size, int match_enable, struct mcx_flow_match *match) 5783 { 5784 struct mcx_cmdq_entry *cqe; 5785 struct mcx_dmamem mxm; 5786 struct mcx_cmd_create_flow_group_in *in; 5787 struct mcx_cmd_create_flow_group_mb_in *mbin; 5788 struct mcx_cmd_create_flow_group_out *out; 5789 struct mcx_flow_group *mfg; 5790 int error; 5791 int token; 5792 5793 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5794 token = mcx_cmdq_token(sc); 5795 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5796 token); 5797 5798 in = mcx_cmdq_in(cqe); 5799 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5800 in->cmd_op_mod = htobe16(0); 5801 5802 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5803 != 0) { 5804 printf("%s: unable to allocate create flow group mailbox\n", 5805 DEVNAME(sc)); 5806 return (-1); 5807 } 5808 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5809 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5810 mbin->cmd_table_id = htobe32(flow_table_id); 5811 mbin->cmd_start_flow_index = htobe32(start); 5812 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5813 5814 mbin->cmd_match_criteria_enable = match_enable; 5815 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5816 5817 mcx_cmdq_mboxes_sign(&mxm, 2); 5818 mcx_cmdq_post(sc, cqe, 0); 5819 error = mcx_cmdq_poll(sc, cqe, 1000); 5820 if (error != 0) { 5821 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5822 goto free; 5823 } 5824 if (mcx_cmdq_verify(cqe) != 0) { 5825 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5826 goto free; 5827 } 5828 5829 out = mcx_cmdq_out(cqe); 5830 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5831 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5832 out->cmd_status, be32toh(out->cmd_syndrome)); 5833 error = -1; 5834 goto free; 5835 } 5836 5837 mfg = &sc->sc_flow_group[group]; 5838 mfg->g_id = mcx_get_id(out->cmd_group_id); 5839 mfg->g_table = flow_table_id; 5840 mfg->g_start = start; 5841 mfg->g_size = size; 5842 5843 free: 5844 mcx_dmamem_free(sc, &mxm); 5845 return (error); 5846 } 5847 5848 static int 5849 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5850 { 5851 struct mcx_cmdq_entry *cqe; 5852 struct mcx_dmamem mxm; 5853 struct mcx_cmd_destroy_flow_group_in *in; 5854 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5855 struct mcx_cmd_destroy_flow_group_out *out; 5856 struct mcx_flow_group *mfg; 5857 int error; 5858 int token; 5859 5860 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5861 token = mcx_cmdq_token(sc); 5862 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5863 5864 in = mcx_cmdq_in(cqe); 5865 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5866 in->cmd_op_mod = htobe16(0); 5867 5868 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5869 &cqe->cq_input_ptr, token) != 0) { 5870 printf("%s: unable to allocate destroy flow group mailbox\n", 5871 DEVNAME(sc)); 5872 return (-1); 5873 } 5874 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5875 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5876 mfg = &sc->sc_flow_group[group]; 5877 mb->cmd_table_id = htobe32(mfg->g_table); 5878 mb->cmd_group_id = htobe32(mfg->g_id); 5879 5880 mcx_cmdq_mboxes_sign(&mxm, 2); 5881 mcx_cmdq_post(sc, cqe, 0); 5882 error = mcx_cmdq_poll(sc, cqe, 1000); 5883 if (error != 0) { 5884 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5885 goto free; 5886 } 5887 if (mcx_cmdq_verify(cqe) != 0) { 5888 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5889 goto free; 5890 } 5891 5892 out = mcx_cmdq_out(cqe); 5893 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5894 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5895 out->cmd_status, be32toh(out->cmd_syndrome)); 5896 error = -1; 5897 goto free; 5898 } 5899 5900 mfg->g_id = -1; 5901 mfg->g_table = -1; 5902 mfg->g_size = 0; 5903 mfg->g_start = 0; 5904 free: 5905 mcx_dmamem_free(sc, &mxm); 5906 return (error); 5907 } 5908 5909 static int 5910 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5911 const uint8_t *macaddr, uint32_t dest) 5912 { 5913 struct mcx_cmdq_entry *cqe; 5914 struct mcx_dmamem mxm; 5915 struct mcx_cmd_set_flow_table_entry_in *in; 5916 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5917 struct mcx_cmd_set_flow_table_entry_out *out; 5918 struct mcx_flow_group *mfg; 5919 uint32_t *pdest; 5920 int error; 5921 int token; 5922 5923 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5924 token = mcx_cmdq_token(sc); 5925 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5926 sizeof(*out), token); 5927 5928 in = mcx_cmdq_in(cqe); 5929 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5930 in->cmd_op_mod = htobe16(0); 5931 5932 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5933 != 0) { 5934 printf("%s: unable to allocate set flow table entry mailbox\n", 5935 DEVNAME(sc)); 5936 return (-1); 5937 } 5938 5939 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5940 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5941 5942 mfg = &sc->sc_flow_group[group]; 5943 mbin->cmd_table_id = htobe32(mfg->g_table); 5944 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5945 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5946 5947 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5948 pdest = (uint32_t *) 5949 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5950 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5951 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5952 *pdest = htobe32(dest); 5953 5954 /* the only thing we match on at the moment is the dest mac address */ 5955 if (macaddr != NULL) { 5956 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5957 ETHER_ADDR_LEN); 5958 } 5959 5960 mcx_cmdq_mboxes_sign(&mxm, 2); 5961 mcx_cmdq_post(sc, cqe, 0); 5962 error = mcx_cmdq_poll(sc, cqe, 1000); 5963 if (error != 0) { 5964 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5965 goto free; 5966 } 5967 if (mcx_cmdq_verify(cqe) != 0) { 5968 printf("%s: set flow table entry command corrupt\n", 5969 DEVNAME(sc)); 5970 goto free; 5971 } 5972 5973 out = mcx_cmdq_out(cqe); 5974 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5975 printf("%s: set flow table entry failed (%x, %x)\n", 5976 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome)); 5977 error = -1; 5978 goto free; 5979 } 5980 5981 free: 5982 mcx_dmamem_free(sc, &mxm); 5983 return (error); 5984 } 5985 5986 static int 5987 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5988 int ethertype, int ip_proto, uint32_t dest) 5989 { 5990 struct mcx_cmdq_entry *cqe; 5991 struct mcx_dmamem mxm; 5992 struct mcx_cmd_set_flow_table_entry_in *in; 5993 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5994 struct mcx_cmd_set_flow_table_entry_out *out; 5995 struct mcx_flow_group *mfg; 5996 uint32_t *pdest; 5997 int error; 5998 int token; 5999 6000 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6001 token = mcx_cmdq_token(sc); 6002 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 6003 sizeof(*out), token); 6004 6005 in = mcx_cmdq_in(cqe); 6006 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 6007 in->cmd_op_mod = htobe16(0); 6008 6009 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 6010 != 0) { 6011 printf("%s: unable to allocate set flow table entry mailbox\n", 6012 DEVNAME(sc)); 6013 return (-1); 6014 } 6015 6016 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6017 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 6018 6019 mfg = &sc->sc_flow_group[group]; 6020 mbin->cmd_table_id = htobe32(mfg->g_table); 6021 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 6022 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 6023 6024 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 6025 pdest = (uint32_t *) 6026 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 6027 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 6028 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 6029 *pdest = htobe32(dest); 6030 6031 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 6032 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 6033 6034 mcx_cmdq_mboxes_sign(&mxm, 2); 6035 mcx_cmdq_post(sc, cqe, 0); 6036 error = mcx_cmdq_poll(sc, cqe, 1000); 6037 if (error != 0) { 6038 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 6039 goto free; 6040 } 6041 if (mcx_cmdq_verify(cqe) != 0) { 6042 printf("%s: set flow table entry command corrupt\n", 6043 DEVNAME(sc)); 6044 goto free; 6045 } 6046 6047 out = mcx_cmdq_out(cqe); 6048 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6049 printf("%s: set flow table entry failed (%x, %x)\n", 6050 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome)); 6051 error = -1; 6052 goto free; 6053 } 6054 6055 free: 6056 mcx_dmamem_free(sc, &mxm); 6057 return (error); 6058 } 6059 6060 static int 6061 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 6062 { 6063 struct mcx_cmdq_entry *cqe; 6064 struct mcx_dmamem mxm; 6065 struct mcx_cmd_delete_flow_table_entry_in *in; 6066 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 6067 struct mcx_cmd_delete_flow_table_entry_out *out; 6068 struct mcx_flow_group *mfg; 6069 int error; 6070 int token; 6071 6072 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6073 token = mcx_cmdq_token(sc); 6074 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 6075 token); 6076 6077 in = mcx_cmdq_in(cqe); 6078 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 6079 in->cmd_op_mod = htobe16(0); 6080 6081 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6082 &cqe->cq_input_ptr, token) != 0) { 6083 printf("%s: unable to allocate " 6084 "delete flow table entry mailbox\n", DEVNAME(sc)); 6085 return (-1); 6086 } 6087 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6088 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 6089 6090 mfg = &sc->sc_flow_group[group]; 6091 mbin->cmd_table_id = htobe32(mfg->g_table); 6092 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 6093 6094 mcx_cmdq_mboxes_sign(&mxm, 2); 6095 mcx_cmdq_post(sc, cqe, 0); 6096 error = mcx_cmdq_poll(sc, cqe, 1000); 6097 if (error != 0) { 6098 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 6099 goto free; 6100 } 6101 if (mcx_cmdq_verify(cqe) != 0) { 6102 printf("%s: delete flow table entry command corrupt\n", 6103 DEVNAME(sc)); 6104 goto free; 6105 } 6106 6107 out = mcx_cmdq_out(cqe); 6108 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6109 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 6110 DEVNAME(sc), group, index, out->cmd_status, 6111 be32toh(out->cmd_syndrome)); 6112 error = -1; 6113 goto free; 6114 } 6115 6116 free: 6117 mcx_dmamem_free(sc, &mxm); 6118 return (error); 6119 } 6120 6121 #if 0 6122 int 6123 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 6124 { 6125 struct mcx_dmamem mxm; 6126 struct mcx_cmdq_entry *cqe; 6127 struct mcx_cmd_query_flow_table_in *in; 6128 struct mcx_cmd_query_flow_table_mb_in *mbin; 6129 struct mcx_cmd_query_flow_table_out *out; 6130 struct mcx_cmd_query_flow_table_mb_out *mbout; 6131 uint8_t token = mcx_cmdq_token(sc); 6132 int error; 6133 int i; 6134 uint8_t *dump; 6135 6136 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6137 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6138 sizeof(*out) + sizeof(*mbout) + 16, token); 6139 6140 in = mcx_cmdq_in(cqe); 6141 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 6142 in->cmd_op_mod = htobe16(0); 6143 6144 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6145 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 6146 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6147 &cqe->cq_output_ptr, token) != 0) { 6148 printf(", unable to allocate query flow table mailboxes\n"); 6149 return (-1); 6150 } 6151 cqe->cq_input_ptr = cqe->cq_output_ptr; 6152 6153 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6154 mbin->cmd_table_type = 0; 6155 mbin->cmd_table_id = htobe32(flow_table_id); 6156 6157 mcx_cmdq_mboxes_sign(&mxm, 1); 6158 6159 mcx_cmdq_post(sc, cqe, 0); 6160 error = mcx_cmdq_poll(sc, cqe, 1000); 6161 if (error != 0) { 6162 printf("%s: query flow table timeout\n", DEVNAME(sc)); 6163 goto free; 6164 } 6165 error = mcx_cmdq_verify(cqe); 6166 if (error != 0) { 6167 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 6168 goto free; 6169 } 6170 6171 out = mcx_cmdq_out(cqe); 6172 switch (out->cmd_status) { 6173 case MCX_CQ_STATUS_OK: 6174 break; 6175 default: 6176 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 6177 out->cmd_status, be32toh(out->cmd_syndrome)); 6178 error = -1; 6179 goto free; 6180 } 6181 6182 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 6183 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6184 dump = (uint8_t *)mbout + 8; 6185 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 6186 printf("%.2x ", dump[i]); 6187 if (i % 16 == 15) 6188 printf("\n"); 6189 } 6190 free: 6191 mcx_cq_mboxes_free(sc, &mxm); 6192 return (error); 6193 } 6194 int 6195 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 6196 { 6197 struct mcx_dmamem mxm; 6198 struct mcx_cmdq_entry *cqe; 6199 struct mcx_cmd_query_flow_table_entry_in *in; 6200 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 6201 struct mcx_cmd_query_flow_table_entry_out *out; 6202 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 6203 uint8_t token = mcx_cmdq_token(sc); 6204 int error; 6205 int i; 6206 uint8_t *dump; 6207 6208 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6209 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6210 sizeof(*out) + sizeof(*mbout) + 16, token); 6211 6212 in = mcx_cmdq_in(cqe); 6213 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 6214 in->cmd_op_mod = htobe16(0); 6215 6216 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6217 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6218 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6219 &cqe->cq_output_ptr, token) != 0) { 6220 printf(", unable to allocate " 6221 "query flow table entry mailboxes\n"); 6222 return (-1); 6223 } 6224 cqe->cq_input_ptr = cqe->cq_output_ptr; 6225 6226 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6227 mbin->cmd_table_type = 0; 6228 mbin->cmd_table_id = htobe32(flow_table_id); 6229 mbin->cmd_flow_index = htobe32(index); 6230 6231 mcx_cmdq_mboxes_sign(&mxm, 1); 6232 6233 mcx_cmdq_post(sc, cqe, 0); 6234 error = mcx_cmdq_poll(sc, cqe, 1000); 6235 if (error != 0) { 6236 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 6237 goto free; 6238 } 6239 error = mcx_cmdq_verify(cqe); 6240 if (error != 0) { 6241 printf("%s: query flow table entry reply corrupt\n", 6242 DEVNAME(sc)); 6243 goto free; 6244 } 6245 6246 out = mcx_cmdq_out(cqe); 6247 switch (out->cmd_status) { 6248 case MCX_CQ_STATUS_OK: 6249 break; 6250 default: 6251 printf("%s: query flow table entry failed (%x/%x)\n", 6252 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome)); 6253 error = -1; 6254 goto free; 6255 } 6256 6257 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 6258 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6259 dump = (uint8_t *)mbout; 6260 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6261 printf("%.2x ", dump[i]); 6262 if (i % 16 == 15) 6263 printf("\n"); 6264 } 6265 6266 free: 6267 mcx_cq_mboxes_free(sc, &mxm); 6268 return (error); 6269 } 6270 6271 int 6272 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 6273 { 6274 struct mcx_dmamem mxm; 6275 struct mcx_cmdq_entry *cqe; 6276 struct mcx_cmd_query_flow_group_in *in; 6277 struct mcx_cmd_query_flow_group_mb_in *mbin; 6278 struct mcx_cmd_query_flow_group_out *out; 6279 struct mcx_cmd_query_flow_group_mb_out *mbout; 6280 uint8_t token = mcx_cmdq_token(sc); 6281 int error; 6282 int i; 6283 uint8_t *dump; 6284 6285 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6286 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6287 sizeof(*out) + sizeof(*mbout) + 16, token); 6288 6289 in = mcx_cmdq_in(cqe); 6290 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 6291 in->cmd_op_mod = htobe16(0); 6292 6293 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 6294 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6295 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6296 &cqe->cq_output_ptr, token) != 0) { 6297 printf(", unable to allocate query flow group mailboxes\n"); 6298 return (-1); 6299 } 6300 cqe->cq_input_ptr = cqe->cq_output_ptr; 6301 6302 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6303 mbin->cmd_table_type = 0; 6304 mbin->cmd_table_id = htobe32(flow_table_id); 6305 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 6306 6307 mcx_cmdq_mboxes_sign(&mxm, 1); 6308 6309 mcx_cmdq_post(sc, cqe, 0); 6310 error = mcx_cmdq_poll(sc, cqe, 1000); 6311 if (error != 0) { 6312 printf("%s: query flow group timeout\n", DEVNAME(sc)); 6313 goto free; 6314 } 6315 error = mcx_cmdq_verify(cqe); 6316 if (error != 0) { 6317 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 6318 goto free; 6319 } 6320 6321 out = mcx_cmdq_out(cqe); 6322 switch (out->cmd_status) { 6323 case MCX_CQ_STATUS_OK: 6324 break; 6325 default: 6326 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 6327 out->cmd_status, be32toh(out->cmd_syndrome)); 6328 error = -1; 6329 goto free; 6330 } 6331 6332 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 6333 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6334 dump = (uint8_t *)mbout; 6335 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6336 printf("%.2x ", dump[i]); 6337 if (i % 16 == 15) 6338 printf("\n"); 6339 } 6340 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 6341 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6342 printf("%.2x ", dump[i]); 6343 if (i % 16 == 15) 6344 printf("\n"); 6345 } 6346 6347 free: 6348 mcx_cq_mboxes_free(sc, &mxm); 6349 return (error); 6350 } 6351 6352 static int 6353 mcx_dump_counters(struct mcx_softc *sc) 6354 { 6355 struct mcx_dmamem mxm; 6356 struct mcx_cmdq_entry *cqe; 6357 struct mcx_cmd_query_vport_counters_in *in; 6358 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6359 struct mcx_cmd_query_vport_counters_out *out; 6360 struct mcx_nic_vport_counters *counters; 6361 int error, token; 6362 6363 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6364 token = mcx_cmdq_token(sc); 6365 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6366 sizeof(*out) + sizeof(*counters), token); 6367 6368 in = mcx_cmdq_in(cqe); 6369 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6370 in->cmd_op_mod = htobe16(0); 6371 6372 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6373 &cqe->cq_output_ptr, token) != 0) { 6374 printf(", unable to allocate " 6375 "query nic vport counters mailboxen\n"); 6376 return (-1); 6377 } 6378 cqe->cq_input_ptr = cqe->cq_output_ptr; 6379 6380 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6381 mbin->cmd_clear = 0x80; 6382 6383 mcx_cmdq_mboxes_sign(&mxm, 1); 6384 mcx_cmdq_post(sc, cqe, 0); 6385 6386 error = mcx_cmdq_poll(sc, cqe, 1000); 6387 if (error != 0) { 6388 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6389 goto free; 6390 } 6391 if (mcx_cmdq_verify(cqe) != 0) { 6392 printf("%s: query nic vport counters command corrupt\n", 6393 DEVNAME(sc)); 6394 goto free; 6395 } 6396 6397 out = mcx_cmdq_out(cqe); 6398 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6399 printf("%s: query nic vport counters failed (%x, %x)\n", 6400 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6401 error = -1; 6402 goto free; 6403 } 6404 6405 counters = (struct mcx_nic_vport_counters *) 6406 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6407 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6408 counters->rx_ucast.packets + counters->tx_ucast.packets + 6409 counters->rx_err.packets + counters->tx_err.packets) 6410 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6411 DEVNAME(sc), 6412 betoh64(counters->tx_err.packets), 6413 betoh64(counters->rx_err.packets), 6414 betoh64(counters->tx_ucast.packets), 6415 betoh64(counters->rx_ucast.packets), 6416 betoh64(counters->tx_bcast.packets), 6417 betoh64(counters->rx_bcast.packets)); 6418 free: 6419 mcx_dmamem_free(sc, &mxm); 6420 6421 return (error); 6422 } 6423 6424 static int 6425 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6426 { 6427 struct mcx_dmamem mxm; 6428 struct mcx_cmdq_entry *cqe; 6429 struct mcx_cmd_query_flow_counter_in *in; 6430 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6431 struct mcx_cmd_query_flow_counter_out *out; 6432 struct mcx_counter *counters; 6433 int error, token; 6434 6435 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6436 token = mcx_cmdq_token(sc); 6437 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6438 sizeof(*counters), token); 6439 6440 in = mcx_cmdq_in(cqe); 6441 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6442 in->cmd_op_mod = htobe16(0); 6443 6444 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6445 &cqe->cq_output_ptr, token) != 0) { 6446 printf(", unable to allocate query flow counter mailboxen\n"); 6447 return (-1); 6448 } 6449 cqe->cq_input_ptr = cqe->cq_output_ptr; 6450 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6451 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6452 mbin->cmd_clear = 0x80; 6453 6454 mcx_cmdq_mboxes_sign(&mxm, 1); 6455 mcx_cmdq_post(sc, cqe, 0); 6456 6457 error = mcx_cmdq_poll(sc, cqe, 1000); 6458 if (error != 0) { 6459 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6460 goto free; 6461 } 6462 if (mcx_cmdq_verify(cqe) != 0) { 6463 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6464 goto free; 6465 } 6466 6467 out = mcx_cmdq_out(cqe); 6468 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6469 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6470 out->cmd_status, betoh32(out->cmd_syndrome)); 6471 error = -1; 6472 goto free; 6473 } 6474 6475 counters = (struct mcx_counter *) 6476 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6477 if (counters->packets) 6478 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6479 betoh64(counters->packets)); 6480 free: 6481 mcx_dmamem_free(sc, &mxm); 6482 6483 return (error); 6484 } 6485 6486 #endif 6487 6488 #if NKSTAT > 0 6489 6490 int 6491 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx) 6492 { 6493 struct mcx_dmamem mxm; 6494 struct mcx_cmdq_entry *cqe; 6495 struct mcx_cmd_query_rq_in *in; 6496 struct mcx_cmd_query_rq_out *out; 6497 struct mcx_cmd_query_rq_mb_out *mbout; 6498 uint8_t token = mcx_cmdq_token(sc); 6499 int error; 6500 6501 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6502 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6503 token); 6504 6505 in = mcx_cmdq_in(cqe); 6506 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 6507 in->cmd_op_mod = htobe16(0); 6508 in->cmd_rqn = htobe32(rx->rx_rqn); 6509 6510 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6511 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6512 &cqe->cq_output_ptr, token) != 0) { 6513 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc)); 6514 return (-1); 6515 } 6516 6517 mcx_cmdq_mboxes_sign(&mxm, 1); 6518 6519 mcx_cmdq_post(sc, cqe, 0); 6520 error = mcx_cmdq_poll(sc, cqe, 1000); 6521 if (error != 0) { 6522 printf("%s: query rq timeout\n", DEVNAME(sc)); 6523 goto free; 6524 } 6525 error = mcx_cmdq_verify(cqe); 6526 if (error != 0) { 6527 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 6528 goto free; 6529 } 6530 6531 out = mcx_cmdq_out(cqe); 6532 switch (out->cmd_status) { 6533 case MCX_CQ_STATUS_OK: 6534 break; 6535 default: 6536 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6537 out->cmd_status, be32toh(out->cmd_syndrome)); 6538 error = -1; 6539 goto free; 6540 } 6541 6542 mbout = (struct mcx_cmd_query_rq_mb_out *) 6543 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6544 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx)); 6545 6546 free: 6547 mcx_cq_mboxes_free(sc, &mxm); 6548 return (error); 6549 } 6550 6551 int 6552 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx) 6553 { 6554 struct mcx_dmamem mxm; 6555 struct mcx_cmdq_entry *cqe; 6556 struct mcx_cmd_query_sq_in *in; 6557 struct mcx_cmd_query_sq_out *out; 6558 struct mcx_cmd_query_sq_mb_out *mbout; 6559 uint8_t token = mcx_cmdq_token(sc); 6560 int error; 6561 6562 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6563 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6564 token); 6565 6566 in = mcx_cmdq_in(cqe); 6567 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6568 in->cmd_op_mod = htobe16(0); 6569 in->cmd_sqn = htobe32(tx->tx_sqn); 6570 6571 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6572 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6573 &cqe->cq_output_ptr, token) != 0) { 6574 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc)); 6575 return (-1); 6576 } 6577 6578 mcx_cmdq_mboxes_sign(&mxm, 1); 6579 6580 mcx_cmdq_post(sc, cqe, 0); 6581 error = mcx_cmdq_poll(sc, cqe, 1000); 6582 if (error != 0) { 6583 printf("%s: query sq timeout\n", DEVNAME(sc)); 6584 goto free; 6585 } 6586 error = mcx_cmdq_verify(cqe); 6587 if (error != 0) { 6588 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6589 goto free; 6590 } 6591 6592 out = mcx_cmdq_out(cqe); 6593 switch (out->cmd_status) { 6594 case MCX_CQ_STATUS_OK: 6595 break; 6596 default: 6597 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6598 out->cmd_status, be32toh(out->cmd_syndrome)); 6599 error = -1; 6600 goto free; 6601 } 6602 6603 mbout = (struct mcx_cmd_query_sq_mb_out *) 6604 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6605 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx)); 6606 6607 free: 6608 mcx_cq_mboxes_free(sc, &mxm); 6609 return (error); 6610 } 6611 6612 int 6613 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx) 6614 { 6615 struct mcx_dmamem mxm; 6616 struct mcx_cmdq_entry *cqe; 6617 struct mcx_cmd_query_cq_in *in; 6618 struct mcx_cmd_query_cq_out *out; 6619 struct mcx_cq_ctx *ctx; 6620 uint8_t token = mcx_cmdq_token(sc); 6621 int error; 6622 6623 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6624 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6625 token); 6626 6627 in = mcx_cmdq_in(cqe); 6628 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ); 6629 in->cmd_op_mod = htobe16(0); 6630 in->cmd_cqn = htobe32(cq->cq_n); 6631 6632 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6633 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6634 &cqe->cq_output_ptr, token) != 0) { 6635 printf("%s: unable to allocate query cq mailboxes\n", 6636 DEVNAME(sc)); 6637 return (-1); 6638 } 6639 6640 mcx_cmdq_mboxes_sign(&mxm, 1); 6641 6642 mcx_cmdq_post(sc, cqe, 0); 6643 error = mcx_cmdq_poll(sc, cqe, 1000); 6644 if (error != 0) { 6645 printf("%s: query cq timeout\n", DEVNAME(sc)); 6646 goto free; 6647 } 6648 if (mcx_cmdq_verify(cqe) != 0) { 6649 printf("%s: query cq reply corrupt\n", DEVNAME(sc)); 6650 goto free; 6651 } 6652 6653 out = mcx_cmdq_out(cqe); 6654 switch (out->cmd_status) { 6655 case MCX_CQ_STATUS_OK: 6656 break; 6657 default: 6658 printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc), 6659 out->cmd_status, be32toh(out->cmd_syndrome)); 6660 error = -1; 6661 goto free; 6662 } 6663 6664 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6665 memcpy(cq_ctx, ctx, sizeof(*cq_ctx)); 6666 free: 6667 mcx_dmamem_free(sc, &mxm); 6668 return (error); 6669 } 6670 6671 int 6672 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx) 6673 { 6674 struct mcx_dmamem mxm; 6675 struct mcx_cmdq_entry *cqe; 6676 struct mcx_cmd_query_eq_in *in; 6677 struct mcx_cmd_query_eq_out *out; 6678 struct mcq_eq_ctx *ctx; 6679 uint8_t token = mcx_cmdq_token(sc); 6680 int error; 6681 6682 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6683 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6684 token); 6685 6686 in = mcx_cmdq_in(cqe); 6687 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ); 6688 in->cmd_op_mod = htobe16(0); 6689 in->cmd_eqn = htobe32(eq->eq_n); 6690 6691 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6692 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6693 &cqe->cq_output_ptr, token) != 0) { 6694 printf("%s: unable to allocate query eq mailboxes\n", 6695 DEVNAME(sc)); 6696 return (-1); 6697 } 6698 6699 mcx_cmdq_mboxes_sign(&mxm, 1); 6700 6701 mcx_cmdq_post(sc, cqe, 0); 6702 error = mcx_cmdq_poll(sc, cqe, 1000); 6703 if (error != 0) { 6704 printf("%s: query eq timeout\n", DEVNAME(sc)); 6705 goto free; 6706 } 6707 if (mcx_cmdq_verify(cqe) != 0) { 6708 printf("%s: query eq reply corrupt\n", DEVNAME(sc)); 6709 goto free; 6710 } 6711 6712 out = mcx_cmdq_out(cqe); 6713 switch (out->cmd_status) { 6714 case MCX_CQ_STATUS_OK: 6715 break; 6716 default: 6717 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc), 6718 out->cmd_status, be32toh(out->cmd_syndrome)); 6719 error = -1; 6720 goto free; 6721 } 6722 6723 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6724 memcpy(eq_ctx, ctx, sizeof(*eq_ctx)); 6725 free: 6726 mcx_dmamem_free(sc, &mxm); 6727 return (error); 6728 } 6729 6730 #endif /* NKSTAT > 0 */ 6731 6732 6733 static inline unsigned int 6734 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots) 6735 { 6736 struct mcx_rq_entry *ring, *rqe; 6737 struct mcx_slot *ms; 6738 struct mbuf *m; 6739 uint slot, p, fills; 6740 6741 ring = MCX_DMA_KVA(&rx->rx_rq_mem); 6742 p = rx->rx_prod; 6743 6744 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6745 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE); 6746 6747 slot = (p % (1 << MCX_LOG_RQ_SIZE)); 6748 rqe = ring; 6749 for (fills = 0; fills < nslots; fills++) { 6750 slot = p % (1 << MCX_LOG_RQ_SIZE); 6751 6752 ms = &rx->rx_slots[slot]; 6753 rqe = &ring[slot]; 6754 6755 m = NULL; 6756 MGETHDR(m, M_DONTWAIT, MT_DATA); 6757 if (m == NULL) 6758 break; 6759 6760 MCLGET(m, M_DONTWAIT); 6761 if ((m->m_flags & M_EXT) == 0) { 6762 m_freem(m); 6763 break; 6764 } 6765 6766 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6767 m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz); 6768 m_adj(m, ETHER_ALIGN); 6769 6770 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6771 BUS_DMA_NOWAIT) != 0) { 6772 m_freem(m); 6773 break; 6774 } 6775 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD); 6776 ms->ms_m = m; 6777 6778 be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len); 6779 be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr); 6780 be32enc(&rqe->rqe_lkey, sc->sc_lkey); 6781 6782 p++; 6783 } 6784 6785 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem), 6786 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE); 6787 6788 rx->rx_prod = p; 6789 6790 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6791 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 6792 be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell), 6793 p & MCX_WQ_DOORBELL_MASK); 6794 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 6795 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 6796 6797 return (nslots - fills); 6798 } 6799 6800 static int 6801 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6802 { 6803 u_int slots; 6804 6805 slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6806 if (slots == 0) 6807 return (1); 6808 6809 slots = mcx_rx_fill_slots(sc, rx, slots); 6810 mcx_rxr_put(&rx->rx_rxr, slots); 6811 return (0); 6812 } 6813 6814 void 6815 mcx_refill(void *xrx) 6816 { 6817 struct mcx_rx *rx = xrx; 6818 struct mcx_softc *sc = rx->rx_softc; 6819 6820 mcx_rx_fill(sc, rx); 6821 6822 if (mcx_rxr_inuse(&rx->rx_rxr) == 0) 6823 callout_schedule(&rx->rx_refill, 1); 6824 } 6825 6826 static int 6827 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6828 struct mcx_cq_entry *cqe) 6829 { 6830 struct mcx_slot *ms; 6831 bus_dmamap_t map; 6832 int slot, slots; 6833 6834 slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6835 6836 ms = &tx->tx_slots[slot]; 6837 map = ms->ms_map; 6838 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6839 BUS_DMASYNC_POSTWRITE); 6840 6841 slots = 1; 6842 if (map->dm_nsegs > 1) 6843 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6844 6845 bus_dmamap_unload(sc->sc_dmat, map); 6846 m_freem(ms->ms_m); 6847 ms->ms_m = NULL; 6848 6849 return (slots); 6850 } 6851 6852 static uint64_t 6853 mcx_uptime(void) 6854 { 6855 struct timespec ts; 6856 6857 nanouptime(&ts); 6858 6859 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6860 } 6861 6862 static void 6863 mcx_calibrate_first(struct mcx_softc *sc) 6864 { 6865 struct mcx_calibration *c = &sc->sc_calibration[0]; 6866 int s; 6867 6868 sc->sc_calibration_gen = 0; 6869 6870 s = splhigh(); /* crit_enter? */ 6871 c->c_ubase = mcx_uptime(); 6872 c->c_tbase = mcx_timer(sc); 6873 splx(s); 6874 c->c_ratio = 0; 6875 6876 #if notyet 6877 callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz); 6878 #endif 6879 } 6880 6881 #define MCX_TIMESTAMP_SHIFT 24 6882 6883 static void 6884 mcx_calibrate(void *arg) 6885 { 6886 struct mcx_softc *sc = arg; 6887 struct mcx_calibration *nc, *pc; 6888 uint64_t udiff, tdiff; 6889 unsigned int gen; 6890 int s; 6891 6892 if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING)) 6893 return; 6894 6895 callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz); 6896 6897 gen = sc->sc_calibration_gen; 6898 pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)]; 6899 gen++; 6900 nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)]; 6901 6902 nc->c_uptime = pc->c_ubase; 6903 nc->c_timestamp = pc->c_tbase; 6904 6905 s = splhigh(); /* crit_enter? */ 6906 nc->c_ubase = mcx_uptime(); 6907 nc->c_tbase = mcx_timer(sc); 6908 splx(s); 6909 6910 udiff = nc->c_ubase - nc->c_uptime; 6911 tdiff = nc->c_tbase - nc->c_timestamp; 6912 6913 /* 6914 * udiff is the wall clock time between calibration ticks, 6915 * which should be 32 seconds or 32 billion nanoseconds. if 6916 * we squint, 1 billion nanoseconds is kind of like a 32 bit 6917 * number, so 32 billion should still have a lot of high bits 6918 * spare. we use this space by shifting the nanoseconds up 6919 * 24 bits so we have a nice big number to divide by the 6920 * number of mcx timer ticks. 6921 */ 6922 nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff; 6923 6924 membar_producer(); 6925 sc->sc_calibration_gen = gen; 6926 } 6927 6928 static int 6929 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6930 struct mcx_cq_entry *cqe, struct mcx_mbufq *mq, 6931 const struct mcx_calibration *c) 6932 { 6933 struct ifnet *ifp = &sc->sc_ec.ec_if; 6934 struct mcx_slot *ms; 6935 struct mbuf *m; 6936 uint32_t flags, len; 6937 int slot; 6938 6939 len = be32dec(&cqe->cq_byte_cnt); 6940 slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6941 6942 ms = &rx->rx_slots[slot]; 6943 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD); 6944 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6945 6946 m = ms->ms_m; 6947 ms->ms_m = NULL; 6948 6949 m_set_rcvif(m, &sc->sc_ec.ec_if); 6950 m->m_pkthdr.len = m->m_len = len; 6951 6952 #if 0 6953 if (cqe->cq_rx_hash_type) { 6954 m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash); 6955 m->m_pkthdr.csum_flags |= M_FLOWID; 6956 } 6957 #endif 6958 6959 flags = be32dec(&cqe->cq_flags); 6960 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) { 6961 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 6962 m->m_pkthdr.csum_flags |= M_CSUM_IPv4; 6963 } 6964 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) { 6965 if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx) 6966 m->m_pkthdr.csum_flags |= M_CSUM_TCPv4; 6967 if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx) 6968 m->m_pkthdr.csum_flags |= M_CSUM_TCPv6; 6969 if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx) 6970 m->m_pkthdr.csum_flags |= M_CSUM_UDPv4; 6971 if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx) 6972 m->m_pkthdr.csum_flags |= M_CSUM_UDPv6; 6973 } 6974 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6975 vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6976 } 6977 6978 #if notyet 6979 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) { 6980 uint64_t t = be64dec(&cqe->cq_timestamp); 6981 t -= c->c_timestamp; 6982 t *= c->c_ratio; 6983 t >>= MCX_TIMESTAMP_SHIFT; 6984 t += c->c_uptime; 6985 6986 m->m_pkthdr.ph_timestamp = t; 6987 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6988 } 6989 #endif 6990 6991 MBUFQ_ENQUEUE(mq, m); 6992 6993 return (1); 6994 } 6995 6996 static struct mcx_cq_entry * 6997 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6998 { 6999 struct mcx_cq_entry *cqe; 7000 int next; 7001 7002 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 7003 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 7004 7005 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 7006 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 7007 return (&cqe[next]); 7008 } 7009 7010 return (NULL); 7011 } 7012 7013 static void 7014 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 7015 { 7016 struct mcx_cq_doorbell *db; 7017 bus_size_t offset; 7018 uint32_t val; 7019 uint64_t uval; 7020 7021 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 7022 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 7023 7024 db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell); 7025 7026 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7027 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE); 7028 7029 be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 7030 be32enc(&db->db_arm_ci, val); 7031 7032 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7033 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE); 7034 7035 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL; 7036 7037 uval = (uint64_t)val << 32; 7038 uval |= cq->cq_n; 7039 7040 bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval)); 7041 mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE); 7042 } 7043 7044 void 7045 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 7046 { 7047 struct mcx_rx *rx = &q->q_rx; 7048 struct mcx_tx *tx = &q->q_tx; 7049 struct ifnet *ifp = &sc->sc_ec.ec_if; 7050 const struct mcx_calibration *c; 7051 unsigned int gen; 7052 struct mcx_cq_entry *cqe; 7053 struct mcx_mbufq mq; 7054 struct mbuf *m; 7055 int rxfree, txfree; 7056 7057 MBUFQ_INIT(&mq); 7058 7059 gen = sc->sc_calibration_gen; 7060 membar_consumer(); 7061 c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)]; 7062 7063 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 7064 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD); 7065 7066 rxfree = 0; 7067 txfree = 0; 7068 while ((cqe = mcx_next_cq_entry(sc, cq))) { 7069 uint8_t opcode; 7070 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 7071 switch (opcode) { 7072 case MCX_CQ_ENTRY_OPCODE_REQ: 7073 txfree += mcx_process_txeof(sc, tx, cqe); 7074 break; 7075 case MCX_CQ_ENTRY_OPCODE_SEND: 7076 rxfree += mcx_process_rx(sc, rx, cqe, &mq, c); 7077 break; 7078 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 7079 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 7080 /* uint8_t *cqp = (uint8_t *)cqe; */ 7081 /* printf("%s: cq completion error: %x\n", 7082 DEVNAME(sc), cqp[0x37]); */ 7083 break; 7084 7085 default: 7086 /* printf("%s: cq completion opcode %x??\n", 7087 DEVNAME(sc), opcode); */ 7088 break; 7089 } 7090 7091 cq->cq_cons++; 7092 } 7093 7094 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem), 7095 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD); 7096 7097 if (rxfree > 0) { 7098 mcx_rxr_put(&rx->rx_rxr, rxfree); 7099 while (MBUFQ_FIRST(&mq) != NULL) { 7100 MBUFQ_DEQUEUE(&mq, m); 7101 if_percpuq_enqueue(ifp->if_percpuq, m); 7102 } 7103 7104 mcx_rx_fill(sc, rx); 7105 if (mcx_rxr_inuse(&rx->rx_rxr) == 0) 7106 callout_schedule(&rx->rx_refill, 1); 7107 } 7108 7109 cq->cq_count++; 7110 mcx_arm_cq(sc, cq, q->q_uar); 7111 7112 if (txfree > 0) { 7113 tx->tx_cons += txfree; 7114 if_schedule_deferred_start(ifp); 7115 } 7116 } 7117 7118 7119 static void 7120 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 7121 { 7122 bus_size_t offset; 7123 uint32_t val; 7124 7125 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 7126 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 7127 7128 mcx_wr(sc, offset, val); 7129 mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE); 7130 } 7131 7132 static struct mcx_eq_entry * 7133 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 7134 { 7135 struct mcx_eq_entry *eqe; 7136 int next; 7137 7138 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 7139 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 7140 if ((eqe[next].eq_owner & 1) == 7141 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 7142 eq->eq_cons++; 7143 return (&eqe[next]); 7144 } 7145 return (NULL); 7146 } 7147 7148 int 7149 mcx_admin_intr(void *xsc) 7150 { 7151 struct mcx_softc *sc = (struct mcx_softc *)xsc; 7152 struct mcx_eq *eq = &sc->sc_admin_eq; 7153 struct mcx_eq_entry *eqe; 7154 7155 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7156 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 7157 7158 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 7159 switch (eqe->eq_event_type) { 7160 case MCX_EVENT_TYPE_LAST_WQE: 7161 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 7162 break; 7163 7164 case MCX_EVENT_TYPE_CQ_ERROR: 7165 /* printf("%s: cq error\n", DEVNAME(sc)); */ 7166 break; 7167 7168 case MCX_EVENT_TYPE_CMD_COMPLETION: 7169 /* wakeup probably */ 7170 break; 7171 7172 case MCX_EVENT_TYPE_PORT_CHANGE: 7173 workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL); 7174 break; 7175 7176 default: 7177 /* printf("%s: something happened\n", DEVNAME(sc)); */ 7178 break; 7179 } 7180 } 7181 7182 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7183 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 7184 7185 mcx_arm_eq(sc, eq, sc->sc_uar); 7186 7187 return (1); 7188 } 7189 7190 int 7191 mcx_cq_intr(void *xq) 7192 { 7193 struct mcx_queues *q = (struct mcx_queues *)xq; 7194 struct mcx_softc *sc = q->q_sc; 7195 struct mcx_eq *eq = &q->q_eq; 7196 struct mcx_eq_entry *eqe; 7197 int cqn; 7198 7199 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7200 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD); 7201 7202 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) { 7203 switch (eqe->eq_event_type) { 7204 case MCX_EVENT_TYPE_COMPLETION: 7205 cqn = be32toh(eqe->eq_event_data[6]); 7206 if (cqn == q->q_cq.cq_n) 7207 mcx_process_cq(sc, q, &q->q_cq); 7208 break; 7209 } 7210 } 7211 7212 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem), 7213 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD); 7214 7215 mcx_arm_eq(sc, eq, q->q_uar); 7216 7217 return (1); 7218 } 7219 7220 static void 7221 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 7222 int total) 7223 { 7224 struct mcx_slot *ms; 7225 7226 int i = allocated; 7227 while (i-- > 0) { 7228 ms = &slots[i]; 7229 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 7230 if (ms->ms_m != NULL) 7231 m_freem(ms->ms_m); 7232 } 7233 kmem_free(slots, total * sizeof(*ms)); 7234 } 7235 7236 static int 7237 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 7238 { 7239 struct mcx_rx *rx; 7240 struct mcx_tx *tx; 7241 struct mcx_slot *ms; 7242 int i; 7243 7244 rx = &q->q_rx; 7245 rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE), 7246 KM_SLEEP); 7247 7248 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 7249 ms = &rx->rx_slots[i]; 7250 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 7251 sc->sc_hardmtu, 0, 7252 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 7253 &ms->ms_map) != 0) { 7254 printf("%s: failed to allocate rx dma maps\n", 7255 DEVNAME(sc)); 7256 goto destroy_rx_slots; 7257 } 7258 } 7259 7260 tx = &q->q_tx; 7261 tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE), 7262 KM_SLEEP); 7263 7264 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 7265 ms = &tx->tx_slots[i]; 7266 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 7267 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 7268 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 7269 &ms->ms_map) != 0) { 7270 printf("%s: failed to allocate tx dma maps\n", 7271 DEVNAME(sc)); 7272 goto destroy_tx_slots; 7273 } 7274 } 7275 7276 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 7277 q->q_eq.eq_n) != 0) 7278 goto destroy_tx_slots; 7279 7280 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 7281 != 0) 7282 goto destroy_cq; 7283 7284 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 7285 goto destroy_sq; 7286 7287 return 0; 7288 7289 destroy_sq: 7290 mcx_destroy_sq(sc, tx); 7291 destroy_cq: 7292 mcx_destroy_cq(sc, &q->q_cq); 7293 destroy_tx_slots: 7294 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 7295 tx->tx_slots = NULL; 7296 7297 i = (1 << MCX_LOG_RQ_SIZE); 7298 destroy_rx_slots: 7299 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 7300 rx->rx_slots = NULL; 7301 return ENOMEM; 7302 } 7303 7304 static int 7305 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 7306 { 7307 int i; 7308 int count; 7309 7310 count = 0; 7311 for (i = 0; i < __arraycount(mcx_rss_config); i++) { 7312 if (mcx_rss_config[i].flow_group == group) 7313 count++; 7314 } 7315 7316 return count; 7317 } 7318 7319 static int 7320 mcx_init(struct ifnet *ifp) 7321 { 7322 struct mcx_softc *sc = ifp->if_softc; 7323 struct mcx_rx *rx; 7324 struct mcx_tx *tx; 7325 int i, start, count, flow_group, flow_index; 7326 struct mcx_flow_match match_crit; 7327 struct mcx_rss_rule *rss; 7328 uint32_t dest; 7329 int rqns[MCX_MAX_QUEUES] = { 0 }; 7330 7331 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7332 mcx_stop(ifp, 0); 7333 7334 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 7335 goto down; 7336 7337 for (i = 0; i < sc->sc_nqueues; i++) { 7338 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 7339 goto down; 7340 } 7341 } 7342 7343 /* RSS flow table and flow groups */ 7344 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 7345 &sc->sc_rss_flow_table_id) != 0) 7346 goto down; 7347 7348 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7349 sc->sc_rss_flow_table_id; 7350 7351 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 7352 memset(&match_crit, 0, sizeof(match_crit)); 7353 match_crit.mc_ethertype = 0xffff; 7354 match_crit.mc_ip_proto = 0xff; 7355 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 7356 start = 0; 7357 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 7358 if (count != 0) { 7359 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7360 MCX_FLOW_GROUP_RSS_L4, start, count, 7361 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7362 goto down; 7363 start += count; 7364 } 7365 7366 /* L3 RSS flow group (v4/v6, including fragments) */ 7367 memset(&match_crit, 0, sizeof(match_crit)); 7368 match_crit.mc_ethertype = 0xffff; 7369 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 7370 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7371 MCX_FLOW_GROUP_RSS_L3, start, count, 7372 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7373 goto down; 7374 start += count; 7375 7376 /* non-RSS flow group */ 7377 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 7378 memset(&match_crit, 0, sizeof(match_crit)); 7379 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 7380 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 7381 goto down; 7382 7383 /* Root flow table, matching packets based on mac address */ 7384 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 7385 &sc->sc_mac_flow_table_id) != 0) 7386 goto down; 7387 7388 /* promisc flow group */ 7389 start = 0; 7390 memset(&match_crit, 0, sizeof(match_crit)); 7391 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7392 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 7393 goto down; 7394 sc->sc_promisc_flow_enabled = 0; 7395 start++; 7396 7397 /* all multicast flow group */ 7398 match_crit.mc_dest_mac[0] = 0x01; 7399 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7400 MCX_FLOW_GROUP_ALLMULTI, start, 1, 7401 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7402 goto down; 7403 sc->sc_allmulti_flow_enabled = 0; 7404 start++; 7405 7406 /* mac address matching flow group */ 7407 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 7408 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 7409 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 7410 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 7411 goto down; 7412 7413 /* flow table entries for unicast and broadcast */ 7414 start = 0; 7415 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7416 LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0) 7417 goto down; 7418 start++; 7419 7420 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 7421 etherbroadcastaddr, dest) != 0) 7422 goto down; 7423 start++; 7424 7425 /* multicast entries go after that */ 7426 sc->sc_mcast_flow_base = start; 7427 7428 /* re-add any existing multicast flows */ 7429 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7430 if (sc->sc_mcast_flows[i][0] != 0) { 7431 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 7432 sc->sc_mcast_flow_base + i, 7433 sc->sc_mcast_flows[i], dest); 7434 } 7435 } 7436 7437 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 7438 goto down; 7439 7440 /* 7441 * the RQT can be any size as long as it's a power of two. 7442 * since we also restrict the number of queues to a power of two, 7443 * we can just put each rx queue in once. 7444 */ 7445 for (i = 0; i < sc->sc_nqueues; i++) 7446 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 7447 7448 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) 7449 goto down; 7450 7451 start = 0; 7452 flow_index = 0; 7453 flow_group = -1; 7454 for (i = 0; i < __arraycount(mcx_rss_config); i++) { 7455 rss = &mcx_rss_config[i]; 7456 if (rss->flow_group != flow_group) { 7457 flow_group = rss->flow_group; 7458 flow_index = 0; 7459 } 7460 7461 if (rss->hash_sel == 0) { 7462 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 7463 &sc->sc_tir[i]) != 0) 7464 goto down; 7465 } else { 7466 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 7467 rss->hash_sel, &sc->sc_tir[i]) != 0) 7468 goto down; 7469 } 7470 7471 if (mcx_set_flow_table_entry_proto(sc, flow_group, 7472 flow_index, rss->ethertype, rss->ip_proto, 7473 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 7474 goto down; 7475 flow_index++; 7476 } 7477 7478 for (i = 0; i < sc->sc_nqueues; i++) { 7479 struct mcx_queues *q = &sc->sc_queues[i]; 7480 rx = &q->q_rx; 7481 tx = &q->q_tx; 7482 7483 /* start the queues */ 7484 if (mcx_ready_sq(sc, tx) != 0) 7485 goto down; 7486 7487 if (mcx_ready_rq(sc, rx) != 0) 7488 goto down; 7489 7490 mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 7491 rx->rx_prod = 0; 7492 mcx_rx_fill(sc, rx); 7493 7494 tx->tx_cons = 0; 7495 tx->tx_prod = 0; 7496 } 7497 7498 mcx_calibrate_first(sc); 7499 7500 SET(ifp->if_flags, IFF_RUNNING); 7501 CLR(ifp->if_flags, IFF_OACTIVE); 7502 if_schedule_deferred_start(ifp); 7503 7504 return 0; 7505 down: 7506 mcx_stop(ifp, 0); 7507 return EIO; 7508 } 7509 7510 static void 7511 mcx_stop(struct ifnet *ifp, int disable) 7512 { 7513 struct mcx_softc *sc = ifp->if_softc; 7514 struct mcx_rss_rule *rss; 7515 int group, i, flow_group, flow_index; 7516 7517 CLR(ifp->if_flags, IFF_RUNNING); 7518 7519 /* 7520 * delete flow table entries first, so no packets can arrive 7521 * after the barriers 7522 */ 7523 if (sc->sc_promisc_flow_enabled) 7524 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 7525 if (sc->sc_allmulti_flow_enabled) 7526 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 7527 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 7528 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 7529 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7530 if (sc->sc_mcast_flows[i][0] != 0) { 7531 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 7532 sc->sc_mcast_flow_base + i); 7533 } 7534 } 7535 7536 flow_group = -1; 7537 flow_index = 0; 7538 for (i = 0; i < __arraycount(mcx_rss_config); i++) { 7539 rss = &mcx_rss_config[i]; 7540 if (rss->flow_group != flow_group) { 7541 flow_group = rss->flow_group; 7542 flow_index = 0; 7543 } 7544 7545 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 7546 7547 mcx_destroy_tir(sc, sc->sc_tir[i]); 7548 sc->sc_tir[i] = 0; 7549 7550 flow_index++; 7551 } 7552 7553 for (i = 0; i < sc->sc_nqueues; i++) { 7554 callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL); 7555 } 7556 7557 callout_halt(&sc->sc_calibrate, NULL); 7558 7559 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 7560 if (sc->sc_flow_group[group].g_id != -1) 7561 mcx_destroy_flow_group(sc, group); 7562 } 7563 7564 if (sc->sc_mac_flow_table_id != -1) { 7565 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 7566 sc->sc_mac_flow_table_id = -1; 7567 } 7568 if (sc->sc_rss_flow_table_id != -1) { 7569 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 7570 sc->sc_rss_flow_table_id = -1; 7571 } 7572 if (sc->sc_rqt != -1) { 7573 mcx_destroy_rqt(sc, sc->sc_rqt); 7574 sc->sc_rqt = -1; 7575 } 7576 7577 for (i = 0; i < sc->sc_nqueues; i++) { 7578 struct mcx_queues *q = &sc->sc_queues[i]; 7579 struct mcx_rx *rx = &q->q_rx; 7580 struct mcx_tx *tx = &q->q_tx; 7581 struct mcx_cq *cq = &q->q_cq; 7582 7583 if (rx->rx_rqn != 0) 7584 mcx_destroy_rq(sc, rx); 7585 7586 if (tx->tx_sqn != 0) 7587 mcx_destroy_sq(sc, tx); 7588 7589 if (tx->tx_slots != NULL) { 7590 mcx_free_slots(sc, tx->tx_slots, 7591 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7592 tx->tx_slots = NULL; 7593 } 7594 if (rx->rx_slots != NULL) { 7595 mcx_free_slots(sc, rx->rx_slots, 7596 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7597 rx->rx_slots = NULL; 7598 } 7599 7600 if (cq->cq_n != 0) 7601 mcx_destroy_cq(sc, cq); 7602 } 7603 if (sc->sc_tis != 0) { 7604 mcx_destroy_tis(sc, sc->sc_tis); 7605 sc->sc_tis = 0; 7606 } 7607 } 7608 7609 static int 7610 mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data) 7611 { 7612 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7613 struct ifreq *ifr = (struct ifreq *)data; 7614 struct ethercom *ec = &sc->sc_ec; 7615 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7616 struct ether_multi *enm; 7617 struct ether_multistep step; 7618 int s, i, flags, error = 0; 7619 uint32_t dest; 7620 7621 s = splnet(); 7622 switch (cmd) { 7623 7624 case SIOCADDMULTI: 7625 if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) { 7626 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7627 if (error != 0) { 7628 splx(s); 7629 return (error); 7630 } 7631 7632 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7633 sc->sc_rss_flow_table_id; 7634 7635 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7636 if (sc->sc_mcast_flows[i][0] == 0) { 7637 memcpy(sc->sc_mcast_flows[i], addrlo, 7638 ETHER_ADDR_LEN); 7639 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7640 mcx_set_flow_table_entry_mac(sc, 7641 MCX_FLOW_GROUP_MAC, 7642 sc->sc_mcast_flow_base + i, 7643 sc->sc_mcast_flows[i], dest); 7644 } 7645 break; 7646 } 7647 } 7648 7649 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7650 if (i == MCX_NUM_MCAST_FLOWS) { 7651 SET(ifp->if_flags, IFF_ALLMULTI); 7652 sc->sc_extra_mcast++; 7653 error = ENETRESET; 7654 } 7655 7656 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) { 7657 SET(ifp->if_flags, IFF_ALLMULTI); 7658 error = ENETRESET; 7659 } 7660 } 7661 } 7662 break; 7663 7664 case SIOCDELMULTI: 7665 if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) { 7666 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7667 if (error != 0) { 7668 splx(s); 7669 return (error); 7670 } 7671 7672 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7673 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7674 ETHER_ADDR_LEN) == 0) { 7675 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7676 mcx_delete_flow_table_entry(sc, 7677 MCX_FLOW_GROUP_MAC, 7678 sc->sc_mcast_flow_base + i); 7679 } 7680 sc->sc_mcast_flows[i][0] = 0; 7681 break; 7682 } 7683 } 7684 7685 if (i == MCX_NUM_MCAST_FLOWS) 7686 sc->sc_extra_mcast--; 7687 7688 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7689 sc->sc_extra_mcast == 0) { 7690 flags = 0; 7691 ETHER_LOCK(ec); 7692 ETHER_FIRST_MULTI(step, ec, enm); 7693 while (enm != NULL) { 7694 if (memcmp(enm->enm_addrlo, 7695 enm->enm_addrhi, ETHER_ADDR_LEN)) { 7696 SET(flags, IFF_ALLMULTI); 7697 break; 7698 } 7699 ETHER_NEXT_MULTI(step, enm); 7700 } 7701 ETHER_UNLOCK(ec); 7702 if (!ISSET(flags, IFF_ALLMULTI)) { 7703 CLR(ifp->if_flags, IFF_ALLMULTI); 7704 error = ENETRESET; 7705 } 7706 } 7707 } 7708 break; 7709 7710 default: 7711 error = ether_ioctl(ifp, cmd, data); 7712 } 7713 7714 if (error == ENETRESET) { 7715 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7716 (IFF_UP | IFF_RUNNING)) 7717 mcx_iff(sc); 7718 error = 0; 7719 } 7720 splx(s); 7721 7722 return (error); 7723 } 7724 7725 #if 0 7726 static int 7727 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7728 { 7729 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7730 struct mcx_reg_mcia mcia; 7731 struct mcx_reg_pmlp pmlp; 7732 int offset, error; 7733 7734 /* get module number */ 7735 memset(&pmlp, 0, sizeof(pmlp)); 7736 pmlp.rp_local_port = 1; 7737 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7738 sizeof(pmlp)); 7739 if (error != 0) { 7740 printf("%s: unable to get eeprom module number\n", 7741 DEVNAME(sc)); 7742 return error; 7743 } 7744 7745 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7746 memset(&mcia, 0, sizeof(mcia)); 7747 mcia.rm_l = 0; 7748 mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) & 7749 MCX_PMLP_MODULE_NUM_MASK; 7750 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7751 mcia.rm_page_num = sff->sff_page; 7752 mcia.rm_dev_addr = htobe16(offset); 7753 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7754 7755 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7756 &mcia, sizeof(mcia)); 7757 if (error != 0) { 7758 printf("%s: unable to read eeprom at %x\n", 7759 DEVNAME(sc), offset); 7760 return error; 7761 } 7762 7763 memcpy(sff->sff_data + offset, mcia.rm_data, 7764 MCX_MCIA_EEPROM_BYTES); 7765 } 7766 7767 return 0; 7768 } 7769 #endif 7770 7771 static int 7772 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7773 { 7774 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7775 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7776 case 0: 7777 break; 7778 7779 case EFBIG: 7780 if (m_defrag(m, M_DONTWAIT) != NULL && 7781 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7782 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7783 break; 7784 7785 /* FALLTHROUGH */ 7786 default: 7787 return (1); 7788 } 7789 7790 ms->ms_m = m; 7791 return (0); 7792 } 7793 7794 static void 7795 mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit) 7796 { 7797 struct mcx_softc *sc = ifp->if_softc; 7798 struct mcx_sq_entry *sq, *sqe; 7799 struct mcx_sq_entry_seg *sqs; 7800 struct mcx_slot *ms; 7801 bus_dmamap_t map; 7802 struct mbuf *m; 7803 u_int idx, free, used; 7804 uint64_t *bf; 7805 uint32_t csum; 7806 size_t bf_base; 7807 int i, seg, nseg; 7808 7809 KASSERT(mutex_owned(&tx->tx_lock)); 7810 7811 if ((ifp->if_flags & IFF_RUNNING) == 0) 7812 return; 7813 7814 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7815 7816 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7817 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7818 7819 used = 0; 7820 bf = NULL; 7821 7822 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7823 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE); 7824 7825 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7826 7827 for (;;) { 7828 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7829 SET(ifp->if_flags, IFF_OACTIVE); 7830 break; 7831 } 7832 7833 if (is_transmit) { 7834 m = pcq_get(tx->tx_pcq); 7835 } else { 7836 IFQ_DEQUEUE(&ifp->if_snd, m); 7837 } 7838 if (m == NULL) { 7839 break; 7840 } 7841 7842 sqe = sq + idx; 7843 ms = &tx->tx_slots[idx]; 7844 memset(sqe, 0, sizeof(*sqe)); 7845 7846 /* ctrl segment */ 7847 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7848 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7849 /* always generate a completion event */ 7850 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7851 7852 /* eth segment */ 7853 csum = 0; 7854 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) 7855 csum |= MCX_SQE_L3_CSUM; 7856 if (m->m_pkthdr.csum_flags & 7857 (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) 7858 csum |= MCX_SQE_L4_CSUM; 7859 sqe->sqe_mss_csum = htobe32(csum); 7860 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7861 if (vlan_has_tag(m)) { 7862 struct ether_vlan_header *evh; 7863 evh = (struct ether_vlan_header *) 7864 &sqe->sqe_inline_headers; 7865 7866 m_copydata(m, 0, ETHER_HDR_LEN, evh); 7867 evh->evl_proto = evh->evl_encap_proto; 7868 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7869 evh->evl_tag = htons(vlan_get_tag(m)); 7870 m_adj(m, ETHER_HDR_LEN); 7871 } else { 7872 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7873 sqe->sqe_inline_headers); 7874 m_adj(m, MCX_SQ_INLINE_SIZE); 7875 } 7876 7877 if (mcx_load_mbuf(sc, ms, m) != 0) { 7878 m_freem(m); 7879 if_statinc(ifp, if_oerrors); 7880 continue; 7881 } 7882 bf = (uint64_t *)sqe; 7883 7884 if (ifp->if_bpf != NULL) 7885 bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers, 7886 MCX_SQ_INLINE_SIZE, m, BPF_D_OUT); 7887 7888 map = ms->ms_map; 7889 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7890 BUS_DMASYNC_PREWRITE); 7891 7892 sqe->sqe_ds_sq_num = 7893 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7894 (map->dm_nsegs + 3)); 7895 7896 /* data segment - first wqe has one segment */ 7897 sqs = sqe->sqe_segs; 7898 seg = 0; 7899 nseg = 1; 7900 for (i = 0; i < map->dm_nsegs; i++) { 7901 if (seg == nseg) { 7902 /* next slot */ 7903 idx++; 7904 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7905 idx = 0; 7906 tx->tx_prod++; 7907 used++; 7908 7909 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7910 seg = 0; 7911 nseg = MCX_SQ_SEGS_PER_SLOT; 7912 } 7913 sqs[seg].sqs_byte_count = 7914 htobe32(map->dm_segs[i].ds_len); 7915 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7916 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7917 seg++; 7918 } 7919 7920 idx++; 7921 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7922 idx = 0; 7923 tx->tx_prod++; 7924 used++; 7925 } 7926 7927 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem), 7928 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE); 7929 7930 if (used) { 7931 bus_size_t blueflame; 7932 7933 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7934 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE); 7935 be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell), 7936 tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7937 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem), 7938 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE); 7939 7940 /* 7941 * write the first 64 bits of the last sqe we produced 7942 * to the blue flame buffer 7943 */ 7944 7945 blueflame = bf_base + tx->tx_bf_offset; 7946 bus_space_write_8(sc->sc_memt, sc->sc_memh, 7947 blueflame, *bf); 7948 mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE); 7949 7950 /* next write goes to the other buffer */ 7951 tx->tx_bf_offset ^= sc->sc_bf_size; 7952 } 7953 } 7954 7955 static void 7956 mcx_start(struct ifnet *ifp) 7957 { 7958 struct mcx_softc *sc = ifp->if_softc; 7959 /* mcx_start() always uses TX ring[0] */ 7960 struct mcx_tx *tx = &sc->sc_queues[0].q_tx; 7961 7962 mutex_enter(&tx->tx_lock); 7963 if (!ISSET(ifp->if_flags, IFF_OACTIVE)) { 7964 mcx_send_common_locked(ifp, tx, false); 7965 } 7966 mutex_exit(&tx->tx_lock); 7967 } 7968 7969 static int 7970 mcx_transmit(struct ifnet *ifp, struct mbuf *m) 7971 { 7972 struct mcx_softc *sc = ifp->if_softc; 7973 struct mcx_tx *tx; 7974 7975 tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx; 7976 if (__predict_false(!pcq_put(tx->tx_pcq, m))) { 7977 m_freem(m); 7978 return ENOBUFS; 7979 } 7980 7981 if (mutex_tryenter(&tx->tx_lock)) { 7982 mcx_send_common_locked(ifp, tx, true); 7983 mutex_exit(&tx->tx_lock); 7984 } else { 7985 softint_schedule(tx->tx_softint); 7986 } 7987 7988 return 0; 7989 } 7990 7991 static void 7992 mcx_deferred_transmit(void *arg) 7993 { 7994 struct mcx_tx *tx = arg; 7995 struct mcx_softc *sc = tx->tx_softc; 7996 struct ifnet *ifp = &sc->sc_ec.ec_if; 7997 7998 mutex_enter(&tx->tx_lock); 7999 if (pcq_peek(tx->tx_pcq) != NULL) { 8000 mcx_send_common_locked(ifp, tx, true); 8001 } 8002 mutex_exit(&tx->tx_lock); 8003 } 8004 8005 static void 8006 mcx_watchdog(struct ifnet *ifp) 8007 { 8008 } 8009 8010 static void 8011 mcx_media_add_types(struct mcx_softc *sc) 8012 { 8013 struct mcx_reg_ptys ptys; 8014 int i; 8015 uint32_t proto_cap; 8016 8017 memset(&ptys, 0, sizeof(ptys)); 8018 ptys.rp_local_port = 1; 8019 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8020 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 8021 sizeof(ptys)) != 0) { 8022 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 8023 return; 8024 } 8025 8026 proto_cap = be32toh(ptys.rp_eth_proto_cap); 8027 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) { 8028 const struct mcx_eth_proto_capability *cap; 8029 if (!ISSET(proto_cap, 1U << i)) 8030 continue; 8031 8032 cap = &mcx_eth_cap_map[i]; 8033 if (cap->cap_media == 0) 8034 continue; 8035 8036 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 8037 } 8038 } 8039 8040 static void 8041 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 8042 { 8043 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 8044 struct mcx_reg_ptys ptys; 8045 int i; 8046 uint32_t proto_oper; 8047 uint64_t media_oper; 8048 8049 memset(&ptys, 0, sizeof(ptys)); 8050 ptys.rp_local_port = 1; 8051 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8052 8053 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 8054 sizeof(ptys)) != 0) { 8055 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 8056 return; 8057 } 8058 8059 proto_oper = be32toh(ptys.rp_eth_proto_oper); 8060 8061 media_oper = 0; 8062 8063 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) { 8064 const struct mcx_eth_proto_capability *cap; 8065 if (!ISSET(proto_oper, 1U << i)) 8066 continue; 8067 8068 cap = &mcx_eth_cap_map[i]; 8069 8070 if (cap->cap_media != 0) 8071 media_oper = cap->cap_media; 8072 } 8073 8074 ifmr->ifm_status = IFM_AVALID; 8075 if (proto_oper != 0) { 8076 ifmr->ifm_status |= IFM_ACTIVE; 8077 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 8078 /* txpause, rxpause, duplex? */ 8079 } 8080 } 8081 8082 static int 8083 mcx_media_change(struct ifnet *ifp) 8084 { 8085 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 8086 struct mcx_reg_ptys ptys; 8087 struct mcx_reg_paos paos; 8088 uint32_t media; 8089 int i, error; 8090 8091 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 8092 return EINVAL; 8093 8094 error = 0; 8095 8096 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 8097 /* read ptys to get supported media */ 8098 memset(&ptys, 0, sizeof(ptys)); 8099 ptys.rp_local_port = 1; 8100 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8101 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 8102 &ptys, sizeof(ptys)) != 0) { 8103 printf("%s: unable to read port type/speed\n", 8104 DEVNAME(sc)); 8105 return EIO; 8106 } 8107 8108 media = be32toh(ptys.rp_eth_proto_cap); 8109 } else { 8110 /* map media type */ 8111 media = 0; 8112 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) { 8113 const struct mcx_eth_proto_capability *cap; 8114 8115 cap = &mcx_eth_cap_map[i]; 8116 if (cap->cap_media == 8117 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 8118 media = (1 << i); 8119 break; 8120 } 8121 } 8122 } 8123 8124 /* disable the port */ 8125 memset(&paos, 0, sizeof(paos)); 8126 paos.rp_local_port = 1; 8127 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 8128 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 8129 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 8130 sizeof(paos)) != 0) { 8131 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 8132 return EIO; 8133 } 8134 8135 memset(&ptys, 0, sizeof(ptys)); 8136 ptys.rp_local_port = 1; 8137 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 8138 ptys.rp_eth_proto_admin = htobe32(media); 8139 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 8140 sizeof(ptys)) != 0) { 8141 printf("%s: unable to set port media type/speed\n", 8142 DEVNAME(sc)); 8143 error = EIO; 8144 } 8145 8146 /* re-enable the port to start negotiation */ 8147 memset(&paos, 0, sizeof(paos)); 8148 paos.rp_local_port = 1; 8149 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 8150 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 8151 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 8152 sizeof(paos)) != 0) { 8153 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 8154 error = EIO; 8155 } 8156 8157 return error; 8158 } 8159 8160 static void 8161 mcx_port_change(struct work *wk, void *xsc) 8162 { 8163 struct mcx_softc *sc = xsc; 8164 struct ifnet *ifp = &sc->sc_ec.ec_if; 8165 struct mcx_reg_ptys ptys = { 8166 .rp_local_port = 1, 8167 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 8168 }; 8169 int link_state = LINK_STATE_DOWN; 8170 8171 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 8172 sizeof(ptys)) == 0) { 8173 uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper); 8174 uint64_t baudrate = 0; 8175 unsigned int i; 8176 8177 if (proto_oper != 0) 8178 link_state = LINK_STATE_UP; 8179 8180 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) { 8181 const struct mcx_eth_proto_capability *cap; 8182 if (!ISSET(proto_oper, 1U << i)) 8183 continue; 8184 8185 cap = &mcx_eth_cap_map[i]; 8186 if (cap->cap_baudrate == 0) 8187 continue; 8188 8189 baudrate = cap->cap_baudrate; 8190 break; 8191 } 8192 8193 ifp->if_baudrate = baudrate; 8194 } 8195 8196 if (link_state != ifp->if_link_state) { 8197 if_link_state_change(ifp, link_state); 8198 } 8199 } 8200 8201 8202 static inline uint32_t 8203 mcx_rd(struct mcx_softc *sc, bus_size_t r) 8204 { 8205 uint32_t word; 8206 8207 word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r); 8208 8209 return (be32toh(word)); 8210 } 8211 8212 static inline void 8213 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 8214 { 8215 bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 8216 } 8217 8218 static inline void 8219 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 8220 { 8221 #ifndef __NetBSD__ 8222 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 8223 #endif 8224 } 8225 8226 static uint64_t 8227 mcx_timer(struct mcx_softc *sc) 8228 { 8229 uint32_t hi, lo, ni; 8230 8231 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8232 for (;;) { 8233 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 8234 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 8235 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 8236 8237 if (ni == hi) 8238 break; 8239 8240 hi = ni; 8241 } 8242 8243 return (((uint64_t)hi << 32) | (uint64_t)lo); 8244 } 8245 8246 static int 8247 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 8248 bus_size_t size, u_int align) 8249 { 8250 mxm->mxm_size = size; 8251 8252 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 8253 mxm->mxm_size, 0, 8254 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 8255 &mxm->mxm_map) != 0) 8256 return (1); 8257 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 8258 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 8259 BUS_DMA_WAITOK) != 0) 8260 goto destroy; 8261 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 8262 mxm->mxm_size, &mxm->mxm_kva, 8263 BUS_DMA_WAITOK | BUS_DMA_COHERENT) != 0) 8264 goto free; 8265 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 8266 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 8267 goto unmap; 8268 8269 mcx_dmamem_zero(mxm); 8270 8271 return (0); 8272 unmap: 8273 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8274 free: 8275 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8276 destroy: 8277 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8278 return (1); 8279 } 8280 8281 static void 8282 mcx_dmamem_zero(struct mcx_dmamem *mxm) 8283 { 8284 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 8285 } 8286 8287 static void 8288 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 8289 { 8290 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 8291 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 8292 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 8293 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 8294 } 8295 8296 static int 8297 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 8298 { 8299 bus_dma_segment_t *segs; 8300 bus_size_t len = pages * MCX_PAGE_SIZE; 8301 size_t seglen; 8302 8303 segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP); 8304 seglen = sizeof(*segs) * pages; 8305 8306 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 8307 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 8308 goto free_segs; 8309 8310 if (mhm->mhm_seg_count < pages) { 8311 size_t nseglen; 8312 8313 mhm->mhm_segs = kmem_alloc( 8314 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP); 8315 8316 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 8317 8318 memcpy(mhm->mhm_segs, segs, nseglen); 8319 8320 kmem_free(segs, seglen); 8321 8322 segs = mhm->mhm_segs; 8323 seglen = nseglen; 8324 } else 8325 mhm->mhm_segs = segs; 8326 8327 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 8328 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 8329 &mhm->mhm_map) != 0) 8330 goto free_dmamem; 8331 8332 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 8333 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 8334 goto destroy; 8335 8336 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8337 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 8338 8339 mhm->mhm_npages = pages; 8340 8341 return (0); 8342 8343 destroy: 8344 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8345 free_dmamem: 8346 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8347 free_segs: 8348 kmem_free(segs, seglen); 8349 mhm->mhm_segs = NULL; 8350 8351 return (-1); 8352 } 8353 8354 static void 8355 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 8356 { 8357 if (mhm->mhm_npages == 0) 8358 return; 8359 8360 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 8361 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 8362 8363 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 8364 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 8365 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 8366 kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 8367 8368 mhm->mhm_npages = 0; 8369 } 8370 8371 #if NKSTAT > 0 8372 struct mcx_ppcnt { 8373 char name[KSTAT_KV_NAMELEN]; 8374 enum kstat_kv_unit unit; 8375 }; 8376 8377 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 8378 { "Good Tx", KSTAT_KV_U_PACKETS, }, 8379 { "Good Rx", KSTAT_KV_U_PACKETS, }, 8380 { "FCS errs", KSTAT_KV_U_PACKETS, }, 8381 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8382 { "Good Tx", KSTAT_KV_U_BYTES, }, 8383 { "Good Rx", KSTAT_KV_U_BYTES, }, 8384 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 8385 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 8386 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 8387 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 8388 { "In Range Len", KSTAT_KV_U_PACKETS, }, 8389 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 8390 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 8391 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 8392 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 8393 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 8394 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 8395 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8396 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8397 }; 8398 CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 8399 8400 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 8401 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 8402 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 8403 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 8404 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 8405 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 8406 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 8407 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 8408 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 8409 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 8410 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 8411 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 8412 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 8413 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 8414 }; 8415 CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 8416 8417 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 8418 { "Drop Events", KSTAT_KV_U_PACKETS, }, 8419 { "Octets", KSTAT_KV_U_BYTES, }, 8420 { "Packets", KSTAT_KV_U_PACKETS, }, 8421 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 8422 { "Multicasts", KSTAT_KV_U_PACKETS, }, 8423 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 8424 { "Undersize", KSTAT_KV_U_PACKETS, }, 8425 { "Oversize", KSTAT_KV_U_PACKETS, }, 8426 { "Fragments", KSTAT_KV_U_PACKETS, }, 8427 { "Jabbers", KSTAT_KV_U_PACKETS, }, 8428 { "Collisions", KSTAT_KV_U_NONE, }, 8429 { "64B", KSTAT_KV_U_PACKETS, }, 8430 { "65-127B", KSTAT_KV_U_PACKETS, }, 8431 { "128-255B", KSTAT_KV_U_PACKETS, }, 8432 { "256-511B", KSTAT_KV_U_PACKETS, }, 8433 { "512-1023B", KSTAT_KV_U_PACKETS, }, 8434 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 8435 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 8436 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 8437 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 8438 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 8439 }; 8440 CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 8441 8442 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 8443 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 8444 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 8445 { "Single Colls", KSTAT_KV_U_PACKETS, }, 8446 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 8447 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 8448 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 8449 { "Late Colls", KSTAT_KV_U_NONE, }, 8450 { "Exess Colls", KSTAT_KV_U_NONE, }, 8451 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 8452 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 8453 { "Too Long", KSTAT_KV_U_PACKETS, }, 8454 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 8455 { "Symbol Errs", KSTAT_KV_U_NONE, }, 8456 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 8457 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8458 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8459 }; 8460 CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 8461 8462 struct mcx_kstat_ppcnt { 8463 const char *ksp_name; 8464 const struct mcx_ppcnt *ksp_tpl; 8465 unsigned int ksp_n; 8466 uint8_t ksp_grp; 8467 }; 8468 8469 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 8470 .ksp_name = "ieee802.3", 8471 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 8472 .ksp_n = __arraycount(mcx_ppcnt_ieee8023_tpl), 8473 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 8474 }; 8475 8476 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 8477 .ksp_name = "rfc2863", 8478 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 8479 .ksp_n = __arraycount(mcx_ppcnt_rfc2863_tpl), 8480 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 8481 }; 8482 8483 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 8484 .ksp_name = "rfc2819", 8485 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 8486 .ksp_n = __arraycount(mcx_ppcnt_rfc2819_tpl), 8487 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 8488 }; 8489 8490 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 8491 .ksp_name = "rfc3635", 8492 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 8493 .ksp_n = __arraycount(mcx_ppcnt_rfc3635_tpl), 8494 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 8495 }; 8496 8497 static int mcx_kstat_ppcnt_read(struct kstat *); 8498 8499 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 8500 static void mcx_kstat_attach_queues(struct mcx_softc *sc); 8501 8502 static struct kstat * 8503 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 8504 const struct mcx_kstat_ppcnt *ksp) 8505 { 8506 struct kstat *ks; 8507 struct kstat_kv *kvs; 8508 unsigned int i; 8509 8510 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 8511 if (ks == NULL) 8512 return (NULL); 8513 8514 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 8515 M_DEVBUF, M_WAITOK); 8516 8517 for (i = 0; i < ksp->ksp_n; i++) { 8518 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 8519 8520 kstat_kv_unit_init(&kvs[i], tpl->name, 8521 KSTAT_KV_T_COUNTER64, tpl->unit); 8522 } 8523 8524 ks->ks_softc = sc; 8525 ks->ks_ptr = (void *)ksp; 8526 ks->ks_data = kvs; 8527 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 8528 ks->ks_read = mcx_kstat_ppcnt_read; 8529 8530 kstat_install(ks); 8531 8532 return (ks); 8533 } 8534 8535 static void 8536 mcx_kstat_attach(struct mcx_softc *sc) 8537 { 8538 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 8539 &mcx_kstat_ppcnt_ieee8023); 8540 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 8541 &mcx_kstat_ppcnt_rfc2863); 8542 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 8543 &mcx_kstat_ppcnt_rfc2819); 8544 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 8545 &mcx_kstat_ppcnt_rfc3635); 8546 8547 mcx_kstat_attach_tmps(sc); 8548 mcx_kstat_attach_queues(sc); 8549 } 8550 8551 static int 8552 mcx_kstat_ppcnt_read(struct kstat *ks) 8553 { 8554 struct mcx_softc *sc = ks->ks_softc; 8555 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 8556 struct mcx_reg_ppcnt ppcnt = { 8557 .ppcnt_grp = ksp->ksp_grp, 8558 .ppcnt_local_port = 1, 8559 }; 8560 struct kstat_kv *kvs = ks->ks_data; 8561 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 8562 unsigned int i; 8563 int rv; 8564 8565 KERNEL_LOCK(); /* XXX */ 8566 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 8567 &ppcnt, sizeof(ppcnt)); 8568 KERNEL_UNLOCK(); 8569 if (rv != 0) 8570 return (EIO); 8571 8572 nanouptime(&ks->ks_updated); 8573 8574 for (i = 0; i < ksp->ksp_n; i++) 8575 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 8576 8577 return (0); 8578 } 8579 8580 struct mcx_kstat_mtmp { 8581 struct kstat_kv ktmp_name; 8582 struct kstat_kv ktmp_temperature; 8583 struct kstat_kv ktmp_threshold_lo; 8584 struct kstat_kv ktmp_threshold_hi; 8585 }; 8586 8587 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 8588 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 8589 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 8590 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 8591 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 8592 }; 8593 8594 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 8595 8596 static int mcx_kstat_mtmp_read(struct kstat *); 8597 8598 static void 8599 mcx_kstat_attach_tmps(struct mcx_softc *sc) 8600 { 8601 struct kstat *ks; 8602 struct mcx_reg_mcam mcam; 8603 struct mcx_reg_mtcap mtcap; 8604 struct mcx_kstat_mtmp *ktmp; 8605 uint64_t map; 8606 unsigned int i, n; 8607 8608 memset(&mtcap, 0, sizeof(mtcap)); 8609 memset(&mcam, 0, sizeof(mcam)); 8610 8611 if (sc->sc_mcam_reg == 0) { 8612 /* no management capabilities */ 8613 return; 8614 } 8615 8616 if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ, 8617 &mcam, sizeof(mcam)) != 0) { 8618 /* unable to check management capabilities? */ 8619 return; 8620 } 8621 8622 if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask, 8623 MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) { 8624 /* no sensor map */ 8625 return; 8626 } 8627 8628 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8629 &mtcap, sizeof(mtcap)) != 0) { 8630 /* unable to find temperature sensors */ 8631 return; 8632 } 8633 8634 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8635 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8636 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8637 8638 n = 0; 8639 map = bemtoh64(&mtcap.mtcap_sensor_map); 8640 for (i = 0; i < sizeof(map) * NBBY; i++) { 8641 if (!ISSET(map, (1ULL << i))) 8642 continue; 8643 8644 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8645 KSTAT_T_KV, 0); 8646 if (ks == NULL) { 8647 /* unable to attach temperature sensor %u, i */ 8648 continue; 8649 } 8650 8651 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8652 *ktmp = mcx_kstat_mtmp_tpl; 8653 8654 ks->ks_data = ktmp; 8655 ks->ks_datalen = sizeof(*ktmp); 8656 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8657 ks->ks_read = mcx_kstat_mtmp_read; 8658 8659 ks->ks_softc = sc; 8660 kstat_install(ks); 8661 8662 sc->sc_kstat_mtmp[n++] = ks; 8663 if (n >= sc->sc_kstat_mtmp_count) 8664 break; 8665 } 8666 } 8667 8668 static uint64_t 8669 mcx_tmp_to_uK(uint16_t *t) 8670 { 8671 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8672 mt *= 1000000 / 8; /* convert to uC */ 8673 mt += 273150000; /* convert to uK */ 8674 8675 return (mt); 8676 } 8677 8678 static int 8679 mcx_kstat_mtmp_read(struct kstat *ks) 8680 { 8681 struct mcx_softc *sc = ks->ks_softc; 8682 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8683 struct mcx_reg_mtmp mtmp; 8684 int rv; 8685 struct timeval updated; 8686 8687 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8688 8689 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8690 return (0); 8691 8692 memset(&mtmp, 0, sizeof(mtmp)); 8693 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8694 8695 KERNEL_LOCK(); /* XXX */ 8696 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8697 &mtmp, sizeof(mtmp)); 8698 KERNEL_UNLOCK(); 8699 if (rv != 0) 8700 return (EIO); 8701 8702 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8703 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8704 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8705 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8706 kstat_kv_temp(&ktmp->ktmp_temperature) = 8707 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8708 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8709 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8710 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8711 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8712 8713 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8714 8715 return (0); 8716 } 8717 8718 struct mcx_queuestat { 8719 char name[KSTAT_KV_NAMELEN]; 8720 enum kstat_kv_type type; 8721 }; 8722 8723 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = { 8724 { "RQ SW prod", KSTAT_KV_T_COUNTER64 }, 8725 { "RQ HW prod", KSTAT_KV_T_COUNTER64 }, 8726 { "RQ HW cons", KSTAT_KV_T_COUNTER64 }, 8727 { "RQ HW state", KSTAT_KV_T_ISTR }, 8728 8729 { "SQ SW prod", KSTAT_KV_T_COUNTER64 }, 8730 { "SQ SW cons", KSTAT_KV_T_COUNTER64 }, 8731 { "SQ HW prod", KSTAT_KV_T_COUNTER64 }, 8732 { "SQ HW cons", KSTAT_KV_T_COUNTER64 }, 8733 { "SQ HW state", KSTAT_KV_T_ISTR }, 8734 8735 { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, 8736 { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, 8737 { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, 8738 { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, 8739 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, 8740 { "CQ HW status", KSTAT_KV_T_ISTR }, 8741 { "CQ HW state", KSTAT_KV_T_ISTR }, 8742 8743 { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, 8744 { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, 8745 { "EQ HW cons", KSTAT_KV_T_COUNTER64 }, 8746 { "EQ HW status", KSTAT_KV_T_ISTR }, 8747 { "EQ HW state", KSTAT_KV_T_ISTR }, 8748 }; 8749 8750 static int mcx_kstat_queue_read(struct kstat *); 8751 8752 static void 8753 mcx_kstat_attach_queues(struct mcx_softc *sc) 8754 { 8755 struct kstat *ks; 8756 struct kstat_kv *kvs; 8757 int q, i; 8758 8759 for (q = 0; q < sc->sc_nqueues; q++) { 8760 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, 8761 KSTAT_T_KV, 0); 8762 if (ks == NULL) { 8763 /* unable to attach queue stats %u, q */ 8764 continue; 8765 } 8766 8767 kvs = mallocarray(nitems(mcx_queue_kstat_tpl), 8768 sizeof(*kvs), M_DEVBUF, M_WAITOK); 8769 8770 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) { 8771 const struct mcx_queuestat *tpl = 8772 &mcx_queue_kstat_tpl[i]; 8773 8774 kstat_kv_init(&kvs[i], tpl->name, tpl->type); 8775 } 8776 8777 ks->ks_softc = &sc->sc_queues[q]; 8778 ks->ks_data = kvs; 8779 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs); 8780 ks->ks_read = mcx_kstat_queue_read; 8781 8782 sc->sc_queues[q].q_kstat = ks; 8783 kstat_install(ks); 8784 } 8785 } 8786 8787 static int 8788 mcx_kstat_queue_read(struct kstat *ks) 8789 { 8790 struct mcx_queues *q = ks->ks_softc; 8791 struct mcx_softc *sc = q->q_sc; 8792 struct kstat_kv *kvs = ks->ks_data; 8793 union { 8794 struct mcx_rq_ctx rq; 8795 struct mcx_sq_ctx sq; 8796 struct mcx_cq_ctx cq; 8797 struct mcx_eq_ctx eq; 8798 } u; 8799 const char *text; 8800 int error = 0; 8801 8802 KERNEL_LOCK(); 8803 8804 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) { 8805 error = EIO; 8806 goto out; 8807 } 8808 8809 kstat_kv_u64(kvs++) = q->q_rx.rx_prod; 8810 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter); 8811 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter); 8812 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >> 8813 MCX_RQ_CTX_STATE_SHIFT) { 8814 case MCX_RQ_CTX_STATE_RST: 8815 text = "RST"; 8816 break; 8817 case MCX_RQ_CTX_STATE_RDY: 8818 text = "RDY"; 8819 break; 8820 case MCX_RQ_CTX_STATE_ERR: 8821 text = "ERR"; 8822 break; 8823 default: 8824 text = "unknown"; 8825 break; 8826 } 8827 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8828 kvs++; 8829 8830 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) { 8831 error = EIO; 8832 goto out; 8833 } 8834 8835 kstat_kv_u64(kvs++) = q->q_tx.tx_prod; 8836 kstat_kv_u64(kvs++) = q->q_tx.tx_cons; 8837 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter); 8838 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter); 8839 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >> 8840 MCX_SQ_CTX_STATE_SHIFT) { 8841 case MCX_SQ_CTX_STATE_RST: 8842 text = "RST"; 8843 break; 8844 case MCX_SQ_CTX_STATE_RDY: 8845 text = "RDY"; 8846 break; 8847 case MCX_SQ_CTX_STATE_ERR: 8848 text = "ERR"; 8849 break; 8850 default: 8851 text = "unknown"; 8852 break; 8853 } 8854 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8855 kvs++; 8856 8857 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) { 8858 error = EIO; 8859 goto out; 8860 } 8861 8862 kstat_kv_u64(kvs++) = q->q_cq.cq_cons; 8863 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); 8864 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); 8865 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); 8866 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); 8867 8868 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >> 8869 MCX_CQ_CTX_STATUS_SHIFT) { 8870 case MCX_CQ_CTX_STATUS_OK: 8871 text = "OK"; 8872 break; 8873 case MCX_CQ_CTX_STATUS_OVERFLOW: 8874 text = "overflow"; 8875 break; 8876 case MCX_CQ_CTX_STATUS_WRITE_FAIL: 8877 text = "write fail"; 8878 break; 8879 default: 8880 text = "unknown"; 8881 break; 8882 } 8883 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8884 kvs++; 8885 8886 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >> 8887 MCX_CQ_CTX_STATE_SHIFT) { 8888 case MCX_CQ_CTX_STATE_SOLICITED: 8889 text = "solicited"; 8890 break; 8891 case MCX_CQ_CTX_STATE_ARMED: 8892 text = "armed"; 8893 break; 8894 case MCX_CQ_CTX_STATE_FIRED: 8895 text = "fired"; 8896 break; 8897 default: 8898 text = "unknown"; 8899 break; 8900 } 8901 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8902 kvs++; 8903 8904 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { 8905 error = EIO; 8906 goto out; 8907 } 8908 8909 kstat_kv_u64(kvs++) = q->q_eq.eq_cons; 8910 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter); 8911 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter); 8912 8913 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >> 8914 MCX_EQ_CTX_STATUS_SHIFT) { 8915 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE: 8916 text = "write fail"; 8917 break; 8918 case MCX_EQ_CTX_STATUS_OK: 8919 text = "OK"; 8920 break; 8921 default: 8922 text = "unknown"; 8923 break; 8924 } 8925 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8926 kvs++; 8927 8928 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >> 8929 MCX_EQ_CTX_STATE_SHIFT) { 8930 case MCX_EQ_CTX_STATE_ARMED: 8931 text = "armed"; 8932 break; 8933 case MCX_EQ_CTX_STATE_FIRED: 8934 text = "fired"; 8935 break; 8936 default: 8937 text = "unknown"; 8938 break; 8939 } 8940 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8941 kvs++; 8942 8943 nanouptime(&ks->ks_updated); 8944 out: 8945 KERNEL_UNLOCK(); 8946 return (error); 8947 } 8948 8949 #endif /* NKSTAT > 0 */ 8950 8951 static unsigned int 8952 mcx_timecounter_read(struct timecounter *tc) 8953 { 8954 struct mcx_softc *sc = tc->tc_priv; 8955 8956 return (mcx_rd(sc, MCX_INTERNAL_TIMER_L)); 8957 } 8958 8959 static void 8960 mcx_timecounter_attach(struct mcx_softc *sc) 8961 { 8962 struct timecounter *tc = &sc->sc_timecounter; 8963 8964 tc->tc_get_timecount = mcx_timecounter_read; 8965 tc->tc_counter_mask = ~0U; 8966 tc->tc_frequency = sc->sc_khz * 1000; 8967 tc->tc_name = device_xname(sc->sc_dev); 8968 tc->tc_quality = -100; 8969 tc->tc_priv = sc; 8970 8971 tc_init(tc); 8972 } 8973