1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <unistd.h> 6 #include <string.h> 7 #include <stdio.h> 8 9 #include <rte_mempool.h> 10 #include <rte_bus_pci.h> 11 #include <rte_malloc.h> 12 #include <rte_errno.h> 13 14 #include "mlx5_devx_cmds.h" 15 #include "../mlx5_common_log.h" 16 #include "mlx5_common.h" 17 #include "mlx5_common_os.h" 18 #include "mlx5_malloc.h" 19 20 /** 21 * Initialization routine for run-time dependency on external lib. 22 */ 23 void 24 mlx5_glue_constructor(void) 25 { 26 } 27 28 /** 29 * Release PD. Releases a given mlx5_pd object 30 * 31 * @param[in] pd 32 * Pointer to mlx5_pd. 33 * 34 * @return 35 * Zero if pd is released successfully, negative number otherwise. 36 */ 37 int 38 mlx5_os_dealloc_pd(void *pd) 39 { 40 if (!pd) 41 return -EINVAL; 42 mlx5_devx_cmd_destroy(((struct mlx5_pd *)pd)->obj); 43 mlx5_free(pd); 44 return 0; 45 } 46 47 /** 48 * Allocate Protection Domain object and extract its pdn using DV API. 49 * 50 * @param[out] dev 51 * Pointer to the mlx5 device. 52 * 53 * @return 54 * 0 on success, a negative value otherwise. 55 */ 56 int 57 mlx5_os_pd_create(struct mlx5_common_device *cdev) 58 { 59 struct mlx5_pd *pd; 60 61 pd = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pd), 0, SOCKET_ID_ANY); 62 if (!pd) 63 return -1; 64 struct mlx5_devx_obj *obj = mlx5_devx_cmd_alloc_pd(cdev->ctx); 65 if (!obj) { 66 mlx5_free(pd); 67 return -1; 68 } 69 pd->obj = obj; 70 pd->pdn = obj->id; 71 pd->devx_ctx = cdev->ctx; 72 cdev->pd = pd; 73 cdev->pdn = pd->pdn; 74 return 0; 75 } 76 77 /** 78 * Detect if a devx_device_bdf object has identical DBDF values to the 79 * rte_pci_addr found in bus/pci probing. 80 * 81 * @param[in] devx_bdf 82 * Pointer to the devx_device_bdf structure. 83 * @param[in] addr 84 * Pointer to the rte_pci_addr structure. 85 * 86 * @return 87 * 1 on Device match, 0 on mismatch. 88 */ 89 static int 90 mlx5_match_devx_bdf_to_addr(struct devx_device_bdf *devx_bdf, 91 struct rte_pci_addr *addr) 92 { 93 if (addr->domain != (devx_bdf->bus_id >> 8) || 94 addr->bus != (devx_bdf->bus_id & 0xff) || 95 addr->devid != devx_bdf->dev_id || 96 addr->function != devx_bdf->fnc_id) { 97 return 0; 98 } 99 return 1; 100 } 101 102 /** 103 * Detect if a devx_device_bdf object matches the rte_pci_addr 104 * found in bus/pci probing 105 * Compare both the Native/PF BDF and the raw_bdf representing a VF BDF. 106 * 107 * @param[in] devx_bdf 108 * Pointer to the devx_device_bdf structure. 109 * @param[in] addr 110 * Pointer to the rte_pci_addr structure. 111 * 112 * @return 113 * 1 on Device match, 0 on mismatch, rte_errno code on failure. 114 */ 115 static int 116 mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf, 117 struct rte_pci_addr *addr) 118 { 119 int err; 120 struct devx_device mlx5_dev; 121 122 if (mlx5_match_devx_bdf_to_addr(devx_bdf, addr)) 123 return 1; 124 /* 125 * Didn't match on Native/PF BDF, could still match a VF BDF, 126 * check it next. 127 */ 128 err = mlx5_glue->query_device(devx_bdf, &mlx5_dev); 129 if (err) { 130 DRV_LOG(ERR, "query_device failed"); 131 rte_errno = err; 132 return rte_errno; 133 } 134 if (mlx5_match_devx_bdf_to_addr(&mlx5_dev.raw_bdf, addr)) 135 return 1; 136 return 0; 137 } 138 139 /** 140 * Look for DevX device that match to given rte_device. 141 * 142 * @param dev 143 * Pointer to the generic device. 144 * @param devx_list 145 * Pointer to head of DevX devices list. 146 * @param n 147 * Number of devices in given DevX devices list. 148 * 149 * @return 150 * A device match on success, NULL otherwise and rte_errno is set. 151 */ 152 static struct devx_device_bdf * 153 mlx5_os_get_devx_device(struct rte_device *dev, 154 struct devx_device_bdf *devx_list, int n) 155 { 156 struct devx_device_bdf *devx_match = NULL; 157 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev); 158 struct rte_pci_addr *addr = &pci_dev->addr; 159 160 while (n-- > 0) { 161 int ret = mlx5_match_devx_devices_to_addr(devx_list, addr); 162 if (!ret) { 163 devx_list++; 164 continue; 165 } 166 if (ret != 1) { 167 rte_errno = ret; 168 return NULL; 169 } 170 devx_match = devx_list; 171 break; 172 } 173 if (devx_match == NULL) { 174 /* No device matches, just complain and bail out. */ 175 DRV_LOG(WARNING, 176 "No DevX device matches PCI device " PCI_PRI_FMT "," 177 " is DevX Configured?", 178 addr->domain, addr->bus, addr->devid, addr->function); 179 rte_errno = ENOENT; 180 } 181 return devx_match; 182 } 183 184 /** 185 * Function API open device under Windows. 186 * 187 * This function calls the Windows glue APIs to open a device. 188 * 189 * @param cdev 190 * Pointer to mlx5 device structure. 191 * @param classes 192 * Chosen classes come from user device arguments. 193 * 194 * @return 195 * 0 on success, a negative errno value otherwise and rte_errno is set. 196 */ 197 int 198 mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes) 199 { 200 struct devx_device_bdf *devx_bdf_dev = NULL; 201 struct devx_device_bdf *devx_list; 202 struct mlx5_context *mlx5_ctx = NULL; 203 int n; 204 205 if (classes != MLX5_CLASS_ETH) { 206 DRV_LOG(ERR, 207 "The chosen classes are not supported on Windows."); 208 rte_errno = ENOTSUP; 209 return -rte_errno; 210 } 211 errno = 0; 212 devx_list = mlx5_glue->get_device_list(&n); 213 if (devx_list == NULL) { 214 rte_errno = errno ? errno : ENOSYS; 215 DRV_LOG(ERR, "Cannot list devices, is DevX enabled?"); 216 return -rte_errno; 217 } 218 devx_bdf_dev = mlx5_os_get_devx_device(cdev->dev, devx_list, n); 219 if (devx_bdf_dev == NULL) 220 goto error; 221 /* Try to open DevX device with DV. */ 222 mlx5_ctx = mlx5_glue->open_device(devx_bdf_dev); 223 if (mlx5_ctx == NULL) { 224 DRV_LOG(ERR, "Failed to open DevX device."); 225 rte_errno = errno; 226 goto error; 227 } 228 if (mlx5_glue->query_device(devx_bdf_dev, &mlx5_ctx->mlx5_dev)) { 229 DRV_LOG(ERR, "Failed to query device context fields."); 230 rte_errno = errno; 231 goto error; 232 } 233 cdev->config.devx = 1; 234 cdev->ctx = mlx5_ctx; 235 mlx5_glue->free_device_list(devx_list); 236 return 0; 237 error: 238 if (mlx5_ctx != NULL) 239 claim_zero(mlx5_glue->close_device(mlx5_ctx)); 240 mlx5_glue->free_device_list(devx_list); 241 return -rte_errno; 242 } 243 244 /** 245 * Register umem. 246 * 247 * @param[in] ctx 248 * Pointer to context. 249 * @param[in] addr 250 * Pointer to memory start address. 251 * @param[in] size 252 * Size of the memory to register. 253 * @param[out] access 254 * UMEM access type 255 * 256 * @return 257 * umem on successful registration, NULL and errno otherwise 258 */ 259 void * 260 mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access) 261 { 262 struct mlx5_devx_umem *umem; 263 264 umem = mlx5_malloc(MLX5_MEM_ZERO, 265 (sizeof(*umem)), 0, SOCKET_ID_ANY); 266 if (!umem) { 267 errno = ENOMEM; 268 return NULL; 269 } 270 umem->umem_hdl = mlx5_glue->devx_umem_reg(ctx, addr, size, access, 271 &umem->umem_id); 272 if (!umem->umem_hdl) { 273 mlx5_free(umem); 274 return NULL; 275 } 276 umem->addr = addr; 277 return umem; 278 } 279 280 /** 281 * Deregister umem. 282 * 283 * @param[in] pumem 284 * Pointer to umem. 285 * 286 * @return 287 * 0 on successful release, negative number otherwise 288 */ 289 int 290 mlx5_os_umem_dereg(void *pumem) 291 { 292 struct mlx5_devx_umem *umem; 293 int err = 0; 294 295 if (!pumem) 296 return err; 297 umem = pumem; 298 if (umem->umem_hdl) 299 err = mlx5_glue->devx_umem_dereg(umem->umem_hdl); 300 mlx5_free(umem); 301 return err; 302 } 303 304 /** 305 * Register mr. Given protection doamin pointer, pointer to addr and length 306 * register the memory region. 307 * 308 * @param[in] pd 309 * Pointer to protection domain context (type mlx5_pd). 310 * @param[in] addr 311 * Pointer to memory start address (type devx_device_ctx). 312 * @param[in] length 313 * Lengtoh of the memory to register. 314 * @param[out] pmd_mr 315 * pmd_mr struct set with lkey, address, length, pointer to mr object, mkey 316 * 317 * @return 318 * 0 on successful registration, -1 otherwise 319 */ 320 static int 321 mlx5_os_reg_mr(void *pd, 322 void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr) 323 { 324 struct mlx5_devx_mkey_attr mkey_attr; 325 struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd; 326 struct mlx5_hca_attr attr; 327 struct mlx5_devx_obj *mkey; 328 void *obj; 329 330 if (!pd || !addr) { 331 rte_errno = EINVAL; 332 return -1; 333 } 334 if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr)) 335 return -1; 336 obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length, 337 IBV_ACCESS_LOCAL_WRITE); 338 if (!obj) 339 return -1; 340 memset(&mkey_attr, 0, sizeof(mkey_attr)); 341 mkey_attr.addr = (uintptr_t)addr; 342 mkey_attr.size = length; 343 mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id; 344 mkey_attr.pd = mlx5_pd->pdn; 345 if (!haswell_broadwell_cpu) { 346 mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write; 347 mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read; 348 } 349 mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr); 350 if (!mkey) { 351 claim_zero(mlx5_os_umem_dereg(obj)); 352 return -1; 353 } 354 pmd_mr->addr = addr; 355 pmd_mr->len = length; 356 pmd_mr->obj = obj; 357 pmd_mr->mkey = mkey; 358 pmd_mr->lkey = pmd_mr->mkey->id; 359 return 0; 360 } 361 362 /** 363 * De-register mr. 364 * 365 * @param[in] pmd_mr 366 * Pointer to PMD mr object 367 */ 368 static void 369 mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr) 370 { 371 if (pmd_mr && pmd_mr->mkey) 372 claim_zero(mlx5_glue->devx_obj_destroy(pmd_mr->mkey->obj)); 373 if (pmd_mr && pmd_mr->obj) 374 claim_zero(mlx5_os_umem_dereg(pmd_mr->obj)); 375 memset(pmd_mr, 0, sizeof(*pmd_mr)); 376 } 377 378 /** 379 * Set the reg_mr and dereg_mr callbacks. 380 * 381 * @param[out] reg_mr_cb 382 * Pointer to reg_mr func 383 * @param[out] dereg_mr_cb 384 * Pointer to dereg_mr func 385 * 386 */ 387 void 388 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb) 389 { 390 *reg_mr_cb = mlx5_os_reg_mr; 391 *dereg_mr_cb = mlx5_os_dereg_mr; 392 } 393