1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <stdbool.h> 11 12 #include <rte_common.h> 13 #include <rte_spinlock.h> 14 15 #include "efx.h" 16 17 #include "sfc.h" 18 #include "sfc_log.h" 19 #include "sfc_switch.h" 20 21 /** 22 * Switch port registry entry. 23 * 24 * Drivers aware of RTE switch domains also have to maintain RTE switch 25 * port IDs for RTE ethdev instances they operate. These IDs are supposed 26 * to stand for physical interconnect entities, in example, PCIe functions. 27 * 28 * In terms of MAE, a physical interconnect entity can be referred to using 29 * an MPORT selector, that is, a 32-bit value. RTE switch port IDs, in turn, 30 * are 16-bit values, so indirect mapping has to be maintained: 31 * 32 * +--------------------+ +---------------------------------------+ 33 * | RTE switch port ID | ------ | MAE switch port entry | 34 * +--------------------+ | --------------------- | 35 * | | 36 * | Entity (PCIe function) MPORT selector | 37 * | + | 38 * | Port type (independent/representor) | 39 * +---------------------------------------+ 40 * 41 * This mapping comprises a port type to ensure that RTE switch port ID 42 * of a represented entity and that of its representor are different in 43 * the case when the entity gets plugged into DPDK and not into a guest. 44 * 45 * Entry data also comprises RTE ethdev's own MPORT. This value 46 * coincides with the entity MPORT in the case of independent ports. 47 * In the case of representors, this ID is not a selector and refers 48 * to an allocatable object (that is, it's likely to change on RTE 49 * ethdev replug). Flow API backend must use this value rather 50 * than entity_mport to support flow rule action PORT_ID. 51 */ 52 struct sfc_mae_switch_port { 53 TAILQ_ENTRY(sfc_mae_switch_port) switch_domain_ports; 54 55 /** RTE ethdev MPORT */ 56 efx_mport_sel_t ethdev_mport; 57 /** RTE ethdev port ID */ 58 uint16_t ethdev_port_id; 59 60 /** Entity (PCIe function) MPORT selector */ 61 efx_mport_sel_t entity_mport; 62 /** Port type (independent/representor) */ 63 enum sfc_mae_switch_port_type type; 64 /** RTE switch port ID */ 65 uint16_t id; 66 }; 67 68 TAILQ_HEAD(sfc_mae_switch_ports, sfc_mae_switch_port); 69 70 /** 71 * Switch domain registry entry. 72 * 73 * Even if an RTE ethdev instance gets unplugged, the corresponding 74 * entry in the switch port registry will not be removed because the 75 * entity (PCIe function) MPORT is static and cannot change. If this 76 * RTE ethdev gets plugged back, the entry will be reused, and 77 * RTE switch port ID will be the same. 78 */ 79 struct sfc_mae_switch_domain { 80 TAILQ_ENTRY(sfc_mae_switch_domain) entries; 81 82 /** HW switch ID */ 83 struct sfc_hw_switch_id *hw_switch_id; 84 /** The number of ports in the switch port registry */ 85 unsigned int nb_ports; 86 /** Switch port registry */ 87 struct sfc_mae_switch_ports ports; 88 /** RTE switch domain ID allocated for a group of devices */ 89 uint16_t id; 90 }; 91 92 TAILQ_HEAD(sfc_mae_switch_domains, sfc_mae_switch_domain); 93 94 /** 95 * MAE representation of RTE switch infrastructure. 96 * 97 * It is possible that an RTE flow API client tries to insert a rule 98 * referencing an RTE ethdev deployed on top of a different physical 99 * device (it may belong to the same vendor or not). This particular 100 * driver/engine cannot support this and has to turn down such rules. 101 * 102 * Technically, it's HW switch identifier which, if queried for each 103 * RTE ethdev instance, indicates relationship between the instances. 104 * In the meantime, RTE flow API clients also need to somehow figure 105 * out relationship between RTE ethdev instances in advance. 106 * 107 * The concept of RTE switch domains resolves this issue. The driver 108 * maintains a static list of switch domains which is easy to browse, 109 * and each RTE ethdev fills RTE switch parameters in device 110 * information structure which is made available to clients. 111 * 112 * Even if all RTE ethdev instances belonging to a switch domain get 113 * unplugged, the corresponding entry in the switch domain registry 114 * will not be removed because the corresponding HW switch exists 115 * regardless of its ports being plugged to DPDK or kept aside. 116 * If a port gets plugged back to DPDK, the corresponding 117 * RTE ethdev will indicate the same RTE switch domain ID. 118 */ 119 struct sfc_mae_switch { 120 /** A lock to protect the whole structure */ 121 rte_spinlock_t lock; 122 /** Switch domain registry */ 123 struct sfc_mae_switch_domains domains; 124 }; 125 126 static struct sfc_mae_switch sfc_mae_switch = { 127 .lock = RTE_SPINLOCK_INITIALIZER, 128 .domains = TAILQ_HEAD_INITIALIZER(sfc_mae_switch.domains), 129 }; 130 131 132 /* This function expects to be called only when the lock is held */ 133 static struct sfc_mae_switch_domain * 134 sfc_mae_find_switch_domain_by_id(uint16_t switch_domain_id) 135 { 136 struct sfc_mae_switch_domain *domain; 137 138 SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); 139 140 TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) { 141 if (domain->id == switch_domain_id) 142 return domain; 143 } 144 145 return NULL; 146 } 147 148 /* This function expects to be called only when the lock is held */ 149 static struct sfc_mae_switch_domain * 150 sfc_mae_find_switch_domain_by_hw_switch_id(const struct sfc_hw_switch_id *id) 151 { 152 struct sfc_mae_switch_domain *domain; 153 154 SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); 155 156 TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) { 157 if (sfc_hw_switch_ids_equal(domain->hw_switch_id, id)) 158 return domain; 159 } 160 161 return NULL; 162 } 163 164 int 165 sfc_mae_assign_switch_domain(struct sfc_adapter *sa, 166 uint16_t *switch_domain_id) 167 { 168 struct sfc_hw_switch_id *hw_switch_id; 169 struct sfc_mae_switch_domain *domain; 170 int rc; 171 172 rte_spinlock_lock(&sfc_mae_switch.lock); 173 174 rc = sfc_hw_switch_id_init(sa, &hw_switch_id); 175 if (rc != 0) 176 goto fail_hw_switch_id_init; 177 178 domain = sfc_mae_find_switch_domain_by_hw_switch_id(hw_switch_id); 179 if (domain != NULL) { 180 sfc_hw_switch_id_fini(sa, hw_switch_id); 181 goto done; 182 } 183 184 domain = rte_zmalloc("sfc_mae_switch_domain", sizeof(*domain), 0); 185 if (domain == NULL) { 186 rc = ENOMEM; 187 goto fail_mem_alloc; 188 } 189 190 /* 191 * This code belongs to driver init path, that is, negation is 192 * done at the end of the path by sfc_eth_dev_init(). RTE APIs 193 * negate error codes, so drop negation here. 194 */ 195 rc = -rte_eth_switch_domain_alloc(&domain->id); 196 if (rc != 0) 197 goto fail_domain_alloc; 198 199 domain->hw_switch_id = hw_switch_id; 200 201 TAILQ_INIT(&domain->ports); 202 203 TAILQ_INSERT_TAIL(&sfc_mae_switch.domains, domain, entries); 204 205 done: 206 *switch_domain_id = domain->id; 207 208 rte_spinlock_unlock(&sfc_mae_switch.lock); 209 210 return 0; 211 212 fail_domain_alloc: 213 rte_free(domain); 214 215 fail_mem_alloc: 216 sfc_hw_switch_id_fini(sa, hw_switch_id); 217 rte_spinlock_unlock(&sfc_mae_switch.lock); 218 219 fail_hw_switch_id_init: 220 return rc; 221 } 222 223 /* This function expects to be called only when the lock is held */ 224 static struct sfc_mae_switch_port * 225 sfc_mae_find_switch_port_by_entity(const struct sfc_mae_switch_domain *domain, 226 const efx_mport_sel_t *entity_mportp, 227 enum sfc_mae_switch_port_type type) 228 { 229 struct sfc_mae_switch_port *port; 230 231 SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); 232 233 TAILQ_FOREACH(port, &domain->ports, switch_domain_ports) { 234 if (port->entity_mport.sel == entity_mportp->sel && 235 port->type == type) 236 return port; 237 } 238 239 return NULL; 240 } 241 242 int 243 sfc_mae_assign_switch_port(uint16_t switch_domain_id, 244 const struct sfc_mae_switch_port_request *req, 245 uint16_t *switch_port_id) 246 { 247 struct sfc_mae_switch_domain *domain; 248 struct sfc_mae_switch_port *port; 249 int rc; 250 251 rte_spinlock_lock(&sfc_mae_switch.lock); 252 253 domain = sfc_mae_find_switch_domain_by_id(switch_domain_id); 254 if (domain == NULL) { 255 rc = EINVAL; 256 goto fail_find_switch_domain_by_id; 257 } 258 259 port = sfc_mae_find_switch_port_by_entity(domain, req->entity_mportp, 260 req->type); 261 if (port != NULL) 262 goto done; 263 264 port = rte_zmalloc("sfc_mae_switch_port", sizeof(*port), 0); 265 if (port == NULL) { 266 rc = ENOMEM; 267 goto fail_mem_alloc; 268 } 269 270 port->entity_mport.sel = req->entity_mportp->sel; 271 port->type = req->type; 272 273 port->id = (domain->nb_ports++); 274 275 TAILQ_INSERT_TAIL(&domain->ports, port, switch_domain_ports); 276 277 done: 278 port->ethdev_mport = *req->ethdev_mportp; 279 port->ethdev_port_id = req->ethdev_port_id; 280 281 *switch_port_id = port->id; 282 283 rte_spinlock_unlock(&sfc_mae_switch.lock); 284 285 return 0; 286 287 fail_mem_alloc: 288 fail_find_switch_domain_by_id: 289 rte_spinlock_unlock(&sfc_mae_switch.lock); 290 return rc; 291 } 292 293 /* This function expects to be called only when the lock is held */ 294 static int 295 sfc_mae_find_switch_port_by_ethdev(uint16_t switch_domain_id, 296 uint16_t ethdev_port_id, 297 efx_mport_sel_t *mport_sel) 298 { 299 struct sfc_mae_switch_domain *domain; 300 struct sfc_mae_switch_port *port; 301 302 SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); 303 304 if (ethdev_port_id == RTE_MAX_ETHPORTS) 305 return EINVAL; 306 307 domain = sfc_mae_find_switch_domain_by_id(switch_domain_id); 308 if (domain == NULL) 309 return EINVAL; 310 311 TAILQ_FOREACH(port, &domain->ports, switch_domain_ports) { 312 if (port->ethdev_port_id == ethdev_port_id) { 313 *mport_sel = port->ethdev_mport; 314 return 0; 315 } 316 } 317 318 return ENOENT; 319 } 320 321 int 322 sfc_mae_switch_port_by_ethdev(uint16_t switch_domain_id, 323 uint16_t ethdev_port_id, 324 efx_mport_sel_t *mport_sel) 325 { 326 int rc; 327 328 rte_spinlock_lock(&sfc_mae_switch.lock); 329 rc = sfc_mae_find_switch_port_by_ethdev(switch_domain_id, 330 ethdev_port_id, mport_sel); 331 rte_spinlock_unlock(&sfc_mae_switch.lock); 332 333 return rc; 334 } 335