1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #include <unistd.h> 6 #include <string.h> 7 #include <stdio.h> 8 9 #include <rte_errno.h> 10 #include <rte_mempool.h> 11 #include <rte_malloc.h> 12 13 #include "mlx5_common.h" 14 #include "mlx5_common_os.h" 15 #include "mlx5_common_utils.h" 16 #include "mlx5_malloc.h" 17 18 int mlx5_common_logtype; 19 20 uint8_t haswell_broadwell_cpu; 21 22 static int 23 mlx5_class_check_handler(__rte_unused const char *key, const char *value, 24 void *opaque) 25 { 26 enum mlx5_class *ret = opaque; 27 28 if (strcmp(value, "vdpa") == 0) { 29 *ret = MLX5_CLASS_VDPA; 30 } else if (strcmp(value, "net") == 0) { 31 *ret = MLX5_CLASS_NET; 32 } else { 33 DRV_LOG(ERR, "Invalid mlx5 class %s. Maybe typo in device" 34 " class argument setting?", value); 35 *ret = MLX5_CLASS_INVALID; 36 } 37 return 0; 38 } 39 40 enum mlx5_class 41 mlx5_class_get(struct rte_devargs *devargs) 42 { 43 struct rte_kvargs *kvlist; 44 const char *key = MLX5_CLASS_ARG_NAME; 45 enum mlx5_class ret = MLX5_CLASS_NET; 46 47 if (devargs == NULL) 48 return ret; 49 kvlist = rte_kvargs_parse(devargs->args, NULL); 50 if (kvlist == NULL) 51 return ret; 52 if (rte_kvargs_count(kvlist, key)) 53 rte_kvargs_process(kvlist, key, mlx5_class_check_handler, &ret); 54 rte_kvargs_free(kvlist); 55 return ret; 56 } 57 58 59 /* In case this is an x86_64 intel processor to check if 60 * we should use relaxed ordering. 61 */ 62 #ifdef RTE_ARCH_X86_64 63 /** 64 * This function returns processor identification and feature information 65 * into the registers. 66 * 67 * @param eax, ebx, ecx, edx 68 * Pointers to the registers that will hold cpu information. 69 * @param level 70 * The main category of information returned. 71 */ 72 static inline void mlx5_cpu_id(unsigned int level, 73 unsigned int *eax, unsigned int *ebx, 74 unsigned int *ecx, unsigned int *edx) 75 { 76 __asm__("cpuid\n\t" 77 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) 78 : "0" (level)); 79 } 80 #endif 81 82 RTE_INIT_PRIO(mlx5_log_init, LOG) 83 { 84 mlx5_common_logtype = rte_log_register("pmd.common.mlx5"); 85 if (mlx5_common_logtype >= 0) 86 rte_log_set_level(mlx5_common_logtype, RTE_LOG_NOTICE); 87 } 88 89 static bool mlx5_common_initialized; 90 91 /** 92 * One time innitialization routine for run-time dependency on glue library 93 * for multiple PMDs. Each mlx5 PMD that depends on mlx5_common module, 94 * must invoke in its constructor. 95 */ 96 void 97 mlx5_common_init(void) 98 { 99 if (mlx5_common_initialized) 100 return; 101 102 mlx5_glue_constructor(); 103 mlx5_common_initialized = true; 104 } 105 106 /** 107 * This function is responsible of initializing the variable 108 * haswell_broadwell_cpu by checking if the cpu is intel 109 * and reading the data returned from mlx5_cpu_id(). 110 * since haswell and broadwell cpus don't have improved performance 111 * when using relaxed ordering we want to check the cpu type before 112 * before deciding whether to enable RO or not. 113 * if the cpu is haswell or broadwell the variable will be set to 1 114 * otherwise it will be 0. 115 */ 116 RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG) 117 { 118 #ifdef RTE_ARCH_X86_64 119 unsigned int broadwell_models[4] = {0x3d, 0x47, 0x4F, 0x56}; 120 unsigned int haswell_models[4] = {0x3c, 0x3f, 0x45, 0x46}; 121 unsigned int i, model, family, brand_id, vendor; 122 unsigned int signature_intel_ebx = 0x756e6547; 123 unsigned int extended_model; 124 unsigned int eax = 0; 125 unsigned int ebx = 0; 126 unsigned int ecx = 0; 127 unsigned int edx = 0; 128 int max_level; 129 130 mlx5_cpu_id(0, &eax, &ebx, &ecx, &edx); 131 vendor = ebx; 132 max_level = eax; 133 if (max_level < 1) { 134 haswell_broadwell_cpu = 0; 135 return; 136 } 137 mlx5_cpu_id(1, &eax, &ebx, &ecx, &edx); 138 model = (eax >> 4) & 0x0f; 139 family = (eax >> 8) & 0x0f; 140 brand_id = ebx & 0xff; 141 extended_model = (eax >> 12) & 0xf0; 142 /* Check if the processor is Haswell or Broadwell */ 143 if (vendor == signature_intel_ebx) { 144 if (family == 0x06) 145 model += extended_model; 146 if (brand_id == 0 && family == 0x6) { 147 for (i = 0; i < RTE_DIM(broadwell_models); i++) 148 if (model == broadwell_models[i]) { 149 haswell_broadwell_cpu = 1; 150 return; 151 } 152 for (i = 0; i < RTE_DIM(haswell_models); i++) 153 if (model == haswell_models[i]) { 154 haswell_broadwell_cpu = 1; 155 return; 156 } 157 } 158 } 159 #endif 160 haswell_broadwell_cpu = 0; 161 } 162 163 /** 164 * Allocate page of door-bells and register it using DevX API. 165 * 166 * @param [in] ctx 167 * Pointer to the device context. 168 * 169 * @return 170 * Pointer to new page on success, NULL otherwise. 171 */ 172 static struct mlx5_devx_dbr_page * 173 mlx5_alloc_dbr_page(void *ctx) 174 { 175 struct mlx5_devx_dbr_page *page; 176 177 /* Allocate space for door-bell page and management data. */ 178 page = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 179 sizeof(struct mlx5_devx_dbr_page), 180 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 181 if (!page) { 182 DRV_LOG(ERR, "cannot allocate dbr page"); 183 return NULL; 184 } 185 /* Register allocated memory. */ 186 page->umem = mlx5_glue->devx_umem_reg(ctx, page->dbrs, 187 MLX5_DBR_PAGE_SIZE, 0); 188 if (!page->umem) { 189 DRV_LOG(ERR, "cannot umem reg dbr page"); 190 mlx5_free(page); 191 return NULL; 192 } 193 return page; 194 } 195 196 /** 197 * Find the next available door-bell, allocate new page if needed. 198 * 199 * @param [in] ctx 200 * Pointer to device context. 201 * @param [in] head 202 * Pointer to the head of dbr pages list. 203 * @param [out] dbr_page 204 * Door-bell page containing the page data. 205 * 206 * @return 207 * Door-bell address offset on success, a negative error value otherwise. 208 */ 209 int64_t 210 mlx5_get_dbr(void *ctx, struct mlx5_dbr_page_list *head, 211 struct mlx5_devx_dbr_page **dbr_page) 212 { 213 struct mlx5_devx_dbr_page *page = NULL; 214 uint32_t i, j; 215 216 LIST_FOREACH(page, head, next) 217 if (page->dbr_count < MLX5_DBR_PER_PAGE) 218 break; 219 if (!page) { /* No page with free door-bell exists. */ 220 page = mlx5_alloc_dbr_page(ctx); 221 if (!page) /* Failed to allocate new page. */ 222 return (-1); 223 LIST_INSERT_HEAD(head, page, next); 224 } 225 /* Loop to find bitmap part with clear bit. */ 226 for (i = 0; 227 i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX; 228 i++) 229 ; /* Empty. */ 230 /* Find the first clear bit. */ 231 MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE); 232 j = rte_bsf64(~page->dbr_bitmap[i]); 233 page->dbr_bitmap[i] |= (UINT64_C(1) << j); 234 page->dbr_count++; 235 *dbr_page = page; 236 return (((i * 64) + j) * sizeof(uint64_t)); 237 } 238 239 /** 240 * Release a door-bell record. 241 * 242 * @param [in] head 243 * Pointer to the head of dbr pages list. 244 * @param [in] umem_id 245 * UMEM ID of page containing the door-bell record to release. 246 * @param [in] offset 247 * Offset of door-bell record in page. 248 * 249 * @return 250 * 0 on success, a negative error value otherwise. 251 */ 252 int32_t 253 mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id, 254 uint64_t offset) 255 { 256 struct mlx5_devx_dbr_page *page = NULL; 257 int ret = 0; 258 259 LIST_FOREACH(page, head, next) 260 /* Find the page this address belongs to. */ 261 if (mlx5_os_get_umem_id(page->umem) == umem_id) 262 break; 263 if (!page) 264 return -EINVAL; 265 page->dbr_count--; 266 if (!page->dbr_count) { 267 /* Page not used, free it and remove from list. */ 268 LIST_REMOVE(page, next); 269 if (page->umem) 270 ret = -mlx5_glue->devx_umem_dereg(page->umem); 271 mlx5_free(page); 272 } else { 273 /* Mark in bitmap that this door-bell is not in use. */ 274 offset /= MLX5_DBR_SIZE; 275 int i = offset / 64; 276 int j = offset % 64; 277 278 page->dbr_bitmap[i] &= ~(UINT64_C(1) << j); 279 } 280 return ret; 281 } 282