xref: /dpdk/drivers/common/mlx5/windows/mlx5_common_os.c (revision 1f37cb2bb46b1fd403faa7c3bf8884e6a4dfde66)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2020 Mellanox Technologies, Ltd
3  */
4 
5 #include <unistd.h>
6 #include <string.h>
7 #include <stdio.h>
8 
9 #include <rte_mempool.h>
10 #include <bus_pci_driver.h>
11 #include <rte_malloc.h>
12 #include <rte_errno.h>
13 
14 #include "mlx5_devx_cmds.h"
15 #include "../mlx5_common_log.h"
16 #include "mlx5_common.h"
17 #include "mlx5_common_os.h"
18 #include "mlx5_malloc.h"
19 
20 /**
21  * Initialization routine for run-time dependency on external lib.
22  */
23 void
mlx5_glue_constructor(void)24 mlx5_glue_constructor(void)
25 {
26 }
27 
28 /**
29  * Validate user arguments for remote PD and CTX.
30  *
31  * @param config
32  *   Pointer to device configuration structure.
33  *
34  * @return
35  *   0 on success, a negative errno value otherwise and rte_errno is set.
36  */
37 int
mlx5_os_remote_pd_and_ctx_validate(struct mlx5_common_dev_config * config)38 mlx5_os_remote_pd_and_ctx_validate(struct mlx5_common_dev_config *config)
39 {
40 	int device_fd = config->device_fd;
41 	int pd_handle = config->pd_handle;
42 
43 	if (pd_handle != MLX5_ARG_UNSET || device_fd != MLX5_ARG_UNSET) {
44 		DRV_LOG(ERR, "Remote PD and CTX is not supported on Windows.");
45 		rte_errno = ENOTSUP;
46 		return -rte_errno;
47 	}
48 	return 0;
49 }
50 
51 /**
52  * Release PD. Releases a given mlx5_pd object
53  *
54  * @param[in] cdev
55  *   Pointer to the mlx5 device.
56  *
57  * @return
58  *   Zero if pd is released successfully, negative number otherwise.
59  */
60 int
mlx5_os_pd_release(struct mlx5_common_device * cdev)61 mlx5_os_pd_release(struct mlx5_common_device *cdev)
62 {
63 	struct mlx5_pd *pd = cdev->pd;
64 
65 	if (!pd)
66 		return -EINVAL;
67 	mlx5_devx_cmd_destroy(pd->obj);
68 	mlx5_free(pd);
69 	return 0;
70 }
71 
72 /**
73  * Allocate Protection Domain object and extract its pdn using DV API.
74  *
75  * @param[out] cdev
76  *   Pointer to the mlx5 device.
77  *
78  * @return
79  *   0 on success, a negative value otherwise.
80  */
81 int
mlx5_os_pd_prepare(struct mlx5_common_device * cdev)82 mlx5_os_pd_prepare(struct mlx5_common_device *cdev)
83 {
84 	struct mlx5_pd *pd;
85 
86 	pd = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pd), 0, SOCKET_ID_ANY);
87 	if (!pd)
88 		return -1;
89 	struct mlx5_devx_obj *obj = mlx5_devx_cmd_alloc_pd(cdev->ctx);
90 	if (!obj) {
91 		mlx5_free(pd);
92 		return -1;
93 	}
94 	pd->obj = obj;
95 	pd->pdn = obj->id;
96 	pd->devx_ctx = cdev->ctx;
97 	cdev->pd = pd;
98 	cdev->pdn = pd->pdn;
99 	return 0;
100 }
101 
102 /**
103  * Detect if a devx_device_bdf object has identical DBDF values to the
104  * rte_pci_addr found in bus/pci probing.
105  *
106  * @param[in] devx_bdf
107  *   Pointer to the devx_device_bdf structure.
108  * @param[in] addr
109  *   Pointer to the rte_pci_addr structure.
110  *
111  * @return
112  *   1 on Device match, 0 on mismatch.
113  */
114 static int
mlx5_match_devx_bdf_to_addr(struct devx_device_bdf * devx_bdf,struct rte_pci_addr * addr)115 mlx5_match_devx_bdf_to_addr(struct devx_device_bdf *devx_bdf,
116 			    struct rte_pci_addr *addr)
117 {
118 	if (addr->domain != (devx_bdf->bus_id >> 8) ||
119 	    addr->bus != (devx_bdf->bus_id & 0xff) ||
120 	    addr->devid != devx_bdf->dev_id ||
121 	    addr->function != devx_bdf->fnc_id) {
122 		return 0;
123 	}
124 	return 1;
125 }
126 
127 /**
128  * Detect if a devx_device_bdf object matches the rte_pci_addr
129  * found in bus/pci probing
130  * Compare both the Native/PF BDF and the raw_bdf representing a VF BDF.
131  *
132  * @param[in] devx_bdf
133  *   Pointer to the devx_device_bdf structure.
134  * @param[in] addr
135  *   Pointer to the rte_pci_addr structure.
136  *
137  * @return
138  *   1 on Device match, 0 on mismatch, rte_errno code on failure.
139  */
140 static int
mlx5_match_devx_devices_to_addr(struct devx_device_bdf * devx_bdf,struct rte_pci_addr * addr)141 mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf,
142 				struct rte_pci_addr *addr)
143 {
144 	int err;
145 	struct devx_device mlx5_dev;
146 
147 	if (mlx5_match_devx_bdf_to_addr(devx_bdf, addr))
148 		return 1;
149 	/*
150 	 * Didn't match on Native/PF BDF, could still match a VF BDF,
151 	 * check it next.
152 	 */
153 	err = mlx5_glue->query_device(devx_bdf, &mlx5_dev);
154 	if (err) {
155 		DRV_LOG(ERR, "query_device failed");
156 		rte_errno = err;
157 		return rte_errno;
158 	}
159 	if (mlx5_match_devx_bdf_to_addr(&mlx5_dev.raw_bdf, addr))
160 		return 1;
161 	return 0;
162 }
163 
164 /**
165  * Look for DevX device that match to given rte_device.
166  *
167  * @param dev
168  *   Pointer to the generic device.
169  * @param devx_list
170  *   Pointer to head of DevX devices list.
171  * @param n
172  *   Number of devices in given DevX devices list.
173  *
174  * @return
175  *   A device match on success, NULL otherwise and rte_errno is set.
176  */
177 static struct devx_device_bdf *
mlx5_os_get_devx_device(struct rte_device * dev,struct devx_device_bdf * devx_list,int n)178 mlx5_os_get_devx_device(struct rte_device *dev,
179 			struct devx_device_bdf *devx_list, int n)
180 {
181 	struct devx_device_bdf *devx_match = NULL;
182 	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev);
183 	struct rte_pci_addr *addr = &pci_dev->addr;
184 
185 	while (n-- > 0) {
186 		int ret = mlx5_match_devx_devices_to_addr(devx_list, addr);
187 		if (!ret) {
188 			devx_list++;
189 			continue;
190 		}
191 		if (ret != 1) {
192 			rte_errno = ret;
193 			return NULL;
194 		}
195 		devx_match = devx_list;
196 		break;
197 	}
198 	if (devx_match == NULL) {
199 		/* No device matches, just complain and bail out. */
200 		DRV_LOG(WARNING,
201 			"No DevX device matches PCI device " PCI_PRI_FMT ","
202 			" is DevX Configured?",
203 			addr->domain, addr->bus, addr->devid, addr->function);
204 		rte_errno = ENOENT;
205 	}
206 	return devx_match;
207 }
208 
209 /**
210  * Function API open device under Windows.
211  *
212  * This function calls the Windows glue APIs to open a device.
213  *
214  * @param cdev
215  *   Pointer to mlx5 device structure.
216  * @param classes
217  *   Chosen classes come from user device arguments.
218  *
219  * @return
220  *   0 on success, a negative errno value otherwise and rte_errno is set.
221  */
222 int
mlx5_os_open_device(struct mlx5_common_device * cdev,uint32_t classes)223 mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
224 {
225 	struct devx_device_bdf *devx_bdf_dev = NULL;
226 	struct devx_device_bdf *devx_list;
227 	struct mlx5_context *mlx5_ctx = NULL;
228 	int n;
229 
230 	if (classes != MLX5_CLASS_ETH && classes != MLX5_CLASS_CRYPTO) {
231 		DRV_LOG(ERR,
232 			"The chosen classes are not supported on Windows.");
233 		rte_errno = ENOTSUP;
234 		return -rte_errno;
235 	}
236 	errno = 0;
237 	devx_list = mlx5_glue->get_device_list(&n);
238 	if (devx_list == NULL) {
239 		rte_errno = errno ? errno : ENOSYS;
240 		DRV_LOG(ERR, "Cannot list devices, is DevX enabled?");
241 		return -rte_errno;
242 	}
243 	devx_bdf_dev = mlx5_os_get_devx_device(cdev->dev, devx_list, n);
244 	if (devx_bdf_dev == NULL)
245 		goto error;
246 	/* Try to open DevX device with DV. */
247 	mlx5_ctx = mlx5_glue->open_device(devx_bdf_dev);
248 	if (mlx5_ctx == NULL) {
249 		DRV_LOG(ERR, "Failed to open DevX device.");
250 		rte_errno = errno;
251 		goto error;
252 	}
253 	if (mlx5_glue->query_device(devx_bdf_dev, &mlx5_ctx->mlx5_dev)) {
254 		DRV_LOG(ERR, "Failed to query device context fields.");
255 		rte_errno = errno;
256 		goto error;
257 	}
258 	cdev->config.devx = 1;
259 	cdev->ctx = mlx5_ctx;
260 	mlx5_glue->free_device_list(devx_list);
261 	return 0;
262 error:
263 	if (mlx5_ctx != NULL)
264 		claim_zero(mlx5_glue->close_device(mlx5_ctx));
265 	mlx5_glue->free_device_list(devx_list);
266 	return -rte_errno;
267 }
268 
269 /**
270  * Register umem.
271  *
272  * @param[in] ctx
273  *   Pointer to context.
274  * @param[in] addr
275  *   Pointer to memory start address.
276  * @param[in] size
277  *   Size of the memory to register.
278  * @param[out] access
279  *   UMEM access type
280  *
281  * @return
282  *   umem on successful registration, NULL and errno otherwise
283  */
284 void *
mlx5_os_umem_reg(void * ctx,void * addr,size_t size,uint32_t access)285 mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access)
286 {
287 	struct mlx5_devx_umem *umem;
288 
289 	umem = mlx5_malloc(MLX5_MEM_ZERO,
290 		(sizeof(*umem)), 0, SOCKET_ID_ANY);
291 	if (!umem) {
292 		errno = ENOMEM;
293 		return NULL;
294 	}
295 	umem->umem_hdl = mlx5_glue->devx_umem_reg(ctx, addr, size, access,
296 		&umem->umem_id);
297 	if (!umem->umem_hdl) {
298 		mlx5_free(umem);
299 		return NULL;
300 	}
301 	umem->addr = addr;
302 	return umem;
303 }
304 
305 /**
306  * Deregister umem.
307  *
308  * @param[in] pumem
309  *   Pointer to umem.
310  *
311  * @return
312  *   0 on successful release, negative number otherwise
313  */
314 int
mlx5_os_umem_dereg(void * pumem)315 mlx5_os_umem_dereg(void *pumem)
316 {
317 	struct mlx5_devx_umem *umem;
318 	int err = 0;
319 
320 	if (!pumem)
321 		return err;
322 	umem = pumem;
323 	if (umem->umem_hdl)
324 		err = mlx5_glue->devx_umem_dereg(umem->umem_hdl);
325 	mlx5_free(umem);
326 	return err;
327 }
328 
329 /**
330  * Register mr. Given protection domain pointer, pointer to addr and length
331  * register the memory region.
332  *
333  * @param[in] pd
334  *   Pointer to protection domain context (type mlx5_pd).
335  * @param[in] addr
336  *   Pointer to memory start address (type devx_device_ctx).
337  * @param[in] length
338  *   Length of the memory to register.
339  * @param[out] pmd_mr
340  *   pmd_mr struct set with lkey, address, length, pointer to mr object, mkey
341  *
342  * @return
343  *   0 on successful registration, -1 otherwise
344  */
345 static int
mlx5_os_reg_mr(void * pd,void * addr,size_t length,struct mlx5_pmd_mr * pmd_mr)346 mlx5_os_reg_mr(void *pd,
347 	       void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
348 {
349 	struct mlx5_devx_mkey_attr mkey_attr;
350 	struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd;
351 	struct mlx5_hca_attr attr;
352 	struct mlx5_devx_obj *mkey;
353 	void *obj;
354 
355 	if (!pd || !addr) {
356 		rte_errno = EINVAL;
357 		return -1;
358 	}
359 	if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr))
360 		return -1;
361 	obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length,
362 			       IBV_ACCESS_LOCAL_WRITE);
363 	if (!obj)
364 		return -1;
365 	memset(&mkey_attr, 0, sizeof(mkey_attr));
366 	mkey_attr.addr = (uintptr_t)addr;
367 	mkey_attr.size = length;
368 	mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id;
369 	mkey_attr.pd = mlx5_pd->pdn;
370 	if (!haswell_broadwell_cpu) {
371 		mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write;
372 		mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read;
373 	}
374 	mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
375 	if (!mkey) {
376 		claim_zero(mlx5_os_umem_dereg(obj));
377 		return -1;
378 	}
379 	pmd_mr->addr = addr;
380 	pmd_mr->len = length;
381 	pmd_mr->obj = obj;
382 	pmd_mr->mkey = mkey;
383 	pmd_mr->lkey = pmd_mr->mkey->id;
384 	return 0;
385 }
386 
387 /**
388  * De-register mr.
389  *
390  * @param[in] pmd_mr
391  *  Pointer to PMD mr object
392  */
393 static void
mlx5_os_dereg_mr(struct mlx5_pmd_mr * pmd_mr)394 mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
395 {
396 	if (!pmd_mr)
397 		return;
398 	if (pmd_mr->mkey)
399 		claim_zero(mlx5_devx_cmd_destroy(pmd_mr->mkey));
400 	if (pmd_mr->obj)
401 		claim_zero(mlx5_os_umem_dereg(pmd_mr->obj));
402 	memset(pmd_mr, 0, sizeof(*pmd_mr));
403 }
404 
405 /**
406  * Set the reg_mr and dereg_mr callbacks.
407  *
408  * @param[out] reg_mr_cb
409  *   Pointer to reg_mr func
410  * @param[out] dereg_mr_cb
411  *   Pointer to dereg_mr func
412  *
413  */
414 void
mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t * reg_mr_cb,mlx5_dereg_mr_t * dereg_mr_cb)415 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
416 {
417 	*reg_mr_cb = mlx5_os_reg_mr;
418 	*dereg_mr_cb = mlx5_os_dereg_mr;
419 }
420 
421 /*
422  * In Windows, no need to wrap the MR, no known issue for it in kernel.
423  * Use the regular function to create direct MR.
424  */
425 int
mlx5_os_wrapped_mkey_create(void * ctx,void * pd,uint32_t pdn,void * addr,size_t length,struct mlx5_pmd_wrapped_mr * wpmd_mr)426 mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr,
427 			    size_t length, struct mlx5_pmd_wrapped_mr *wpmd_mr)
428 {
429 	struct mlx5_pmd_mr pmd_mr = {0};
430 	int ret = mlx5_os_reg_mr(pd, addr, length, &pmd_mr);
431 
432 	(void)pdn;
433 	(void)ctx;
434 	if (ret != 0)
435 		return -1;
436 	wpmd_mr->addr = addr;
437 	wpmd_mr->len = length;
438 	wpmd_mr->obj = pmd_mr.obj;
439 	wpmd_mr->imkey = pmd_mr.mkey;
440 	wpmd_mr->lkey = pmd_mr.mkey->id;
441 	return 0;
442 }
443 
444 void
mlx5_os_wrapped_mkey_destroy(struct mlx5_pmd_wrapped_mr * wpmd_mr)445 mlx5_os_wrapped_mkey_destroy(struct mlx5_pmd_wrapped_mr *wpmd_mr)
446 {
447 	struct mlx5_pmd_mr pmd_mr;
448 
449 	if (!wpmd_mr)
450 		return;
451 	pmd_mr.addr = wpmd_mr->addr;
452 	pmd_mr.len = wpmd_mr->len;
453 	pmd_mr.obj = wpmd_mr->obj;
454 	pmd_mr.mkey = wpmd_mr->imkey;
455 	pmd_mr.lkey = wpmd_mr->lkey;
456 	mlx5_os_dereg_mr(&pmd_mr);
457 	memset(wpmd_mr, 0, sizeof(*wpmd_mr));
458 }
459