1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Netronome Systems, Inc. 3 * All rights reserved. 4 */ 5 6 #include <errno.h> 7 8 #include <malloc.h> 9 #include <time.h> 10 #include <sched.h> 11 12 #include "nfp_cpp.h" 13 #include "nfp_logs.h" 14 #include "nfp6000/nfp6000.h" 15 16 #define MUTEX_LOCKED(interface) ((((uint32_t)(interface)) << 16) | 0x000f) 17 #define MUTEX_UNLOCK(interface) (0 | 0x0000) 18 19 #define MUTEX_IS_LOCKED(value) (((value) & 0xffff) == 0x000f) 20 #define MUTEX_IS_UNLOCKED(value) (((value) & 0xffff) == 0x0000) 21 #define MUTEX_INTERFACE(value) (((value) >> 16) & 0xffff) 22 23 /* 24 * If you need more than 65536 recursive locks, please 25 * rethink your code. 26 */ 27 #define MUTEX_DEPTH_MAX 0xffff 28 29 struct nfp_cpp_mutex { 30 struct nfp_cpp *cpp; 31 uint8_t target; 32 uint16_t depth; 33 unsigned long long address; 34 uint32_t key; 35 unsigned int usage; 36 struct nfp_cpp_mutex *prev, *next; 37 }; 38 39 static int 40 _nfp_cpp_mutex_validate(uint32_t model, int *target, unsigned long long address) 41 { 42 /* Address must be 64-bit aligned */ 43 if (address & 7) 44 return NFP_ERRNO(EINVAL); 45 46 if (NFP_CPP_MODEL_IS_6000(model)) { 47 if (*target != NFP_CPP_TARGET_MU) 48 return NFP_ERRNO(EINVAL); 49 } else { 50 return NFP_ERRNO(EINVAL); 51 } 52 53 return 0; 54 } 55 56 /* 57 * Initialize a mutex location 58 * 59 * The CPP target:address must point to a 64-bit aligned location, and 60 * will initialize 64 bits of data at the location. 61 * 62 * This creates the initial mutex state, as locked by this 63 * nfp_cpp_interface(). 64 * 65 * This function should only be called when setting up 66 * the initial lock state upon boot-up of the system. 67 * 68 * @param mutex NFP CPP Mutex handle 69 * @param target NFP CPP target ID (ie NFP_CPP_TARGET_CLS or 70 * NFP_CPP_TARGET_MU) 71 * @param address Offset into the address space of the NFP CPP target ID 72 * @param key Unique 32-bit value for this mutex 73 * 74 * @return 0 on success, or -1 on failure (and set errno accordingly). 75 */ 76 int 77 nfp_cpp_mutex_init(struct nfp_cpp *cpp, int target, unsigned long long address, 78 uint32_t key) 79 { 80 uint32_t model = nfp_cpp_model(cpp); 81 uint32_t muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 82 int err; 83 84 err = _nfp_cpp_mutex_validate(model, &target, address); 85 if (err < 0) 86 return err; 87 88 err = nfp_cpp_writel(cpp, muw, address + 4, key); 89 if (err < 0) 90 return err; 91 92 err = 93 nfp_cpp_writel(cpp, muw, address + 0, 94 MUTEX_LOCKED(nfp_cpp_interface(cpp))); 95 if (err < 0) 96 return err; 97 98 return 0; 99 } 100 101 /* 102 * Create a mutex handle from an address controlled by a MU Atomic engine 103 * 104 * The CPP target:address must point to a 64-bit aligned location, and 105 * reserve 64 bits of data at the location for use by the handle. 106 * 107 * Only target/address pairs that point to entities that support the 108 * MU Atomic Engine are supported. 109 * 110 * @param cpp NFP CPP handle 111 * @param target NFP CPP target ID (ie NFP_CPP_TARGET_CLS or 112 * NFP_CPP_TARGET_MU) 113 * @param address Offset into the address space of the NFP CPP target ID 114 * @param key 32-bit unique key (must match the key at this location) 115 * 116 * @return A non-NULL struct nfp_cpp_mutex * on success, NULL on failure. 117 */ 118 struct nfp_cpp_mutex * 119 nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target, 120 unsigned long long address, uint32_t key) 121 { 122 uint32_t model = nfp_cpp_model(cpp); 123 struct nfp_cpp_mutex *mutex; 124 uint32_t mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 125 int err; 126 uint32_t tmp; 127 128 /* Look for cached mutex */ 129 for (mutex = cpp->mutex_cache; mutex; mutex = mutex->next) { 130 if (mutex->target == target && mutex->address == address) 131 break; 132 } 133 134 if (mutex) { 135 if (mutex->key == key) { 136 mutex->usage++; 137 return mutex; 138 } 139 140 /* If the key doesn't match... */ 141 return NFP_ERRPTR(EEXIST); 142 } 143 144 err = _nfp_cpp_mutex_validate(model, &target, address); 145 if (err < 0) 146 return NULL; 147 148 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp); 149 if (err < 0) 150 return NULL; 151 152 if (tmp != key) 153 return NFP_ERRPTR(EEXIST); 154 155 mutex = calloc(sizeof(*mutex), 1); 156 if (mutex == NULL) 157 return NFP_ERRPTR(ENOMEM); 158 159 mutex->cpp = cpp; 160 mutex->target = target; 161 mutex->address = address; 162 mutex->key = key; 163 mutex->depth = 0; 164 mutex->usage = 1; 165 166 /* Add mutex to the cache */ 167 if (cpp->mutex_cache) { 168 cpp->mutex_cache->prev = mutex; 169 mutex->next = cpp->mutex_cache; 170 cpp->mutex_cache = mutex; 171 } else { 172 cpp->mutex_cache = mutex; 173 } 174 175 return mutex; 176 } 177 178 struct nfp_cpp * 179 nfp_cpp_mutex_cpp(struct nfp_cpp_mutex *mutex) 180 { 181 return mutex->cpp; 182 } 183 184 uint32_t 185 nfp_cpp_mutex_key(struct nfp_cpp_mutex *mutex) 186 { 187 return mutex->key; 188 } 189 190 uint16_t 191 nfp_cpp_mutex_owner(struct nfp_cpp_mutex *mutex) 192 { 193 uint32_t mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 194 uint32_t value, key; 195 int err; 196 197 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value); 198 if (err < 0) 199 return err; 200 201 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key); 202 if (err < 0) 203 return err; 204 205 if (key != mutex->key) 206 return NFP_ERRNO(EPERM); 207 208 if (MUTEX_IS_LOCKED(value) == 0) 209 return 0; 210 211 return MUTEX_INTERFACE(value); 212 } 213 214 int 215 nfp_cpp_mutex_target(struct nfp_cpp_mutex *mutex) 216 { 217 return mutex->target; 218 } 219 220 uint64_t 221 nfp_cpp_mutex_address(struct nfp_cpp_mutex *mutex) 222 { 223 return mutex->address; 224 } 225 226 /* 227 * Free a mutex handle - does not alter the lock state 228 * 229 * @param mutex NFP CPP Mutex handle 230 */ 231 void 232 nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex) 233 { 234 mutex->usage--; 235 if (mutex->usage > 0) 236 return; 237 238 /* Remove mutex from the cache */ 239 if (mutex->next) 240 mutex->next->prev = mutex->prev; 241 if (mutex->prev) 242 mutex->prev->next = mutex->next; 243 244 /* If mutex->cpp == NULL, something broke */ 245 if (mutex->cpp && mutex == mutex->cpp->mutex_cache) 246 mutex->cpp->mutex_cache = mutex->next; 247 248 free(mutex); 249 } 250 251 /* 252 * Lock a mutex handle, using the NFP MU Atomic Engine 253 * 254 * @param mutex NFP CPP Mutex handle 255 * 256 * @return 0 on success, or -1 on failure (and set errno accordingly). 257 */ 258 int 259 nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex) 260 { 261 int err; 262 time_t warn_at = time(NULL) + 15; 263 264 while ((err = nfp_cpp_mutex_trylock(mutex)) != 0) { 265 /* If errno != EBUSY, then the lock was damaged */ 266 if (err < 0 && errno != EBUSY) 267 return err; 268 if (time(NULL) >= warn_at) { 269 PMD_DRV_LOG(ERR, "Warning: waiting for NFP mutex usage:%u depth:%hd] target:%d addr:%llx key:%08x]", 270 mutex->usage, mutex->depth, mutex->target, 271 mutex->address, mutex->key); 272 warn_at = time(NULL) + 60; 273 } 274 sched_yield(); 275 } 276 return 0; 277 } 278 279 /* 280 * Unlock a mutex handle, using the NFP MU Atomic Engine 281 * 282 * @param mutex NFP CPP Mutex handle 283 * 284 * @return 0 on success, or -1 on failure (and set errno accordingly). 285 */ 286 int 287 nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex) 288 { 289 uint32_t muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 290 uint32_t mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 291 struct nfp_cpp *cpp = mutex->cpp; 292 uint32_t key, value; 293 uint16_t interface = nfp_cpp_interface(cpp); 294 int err; 295 296 if (mutex->depth > 1) { 297 mutex->depth--; 298 return 0; 299 } 300 301 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value); 302 if (err < 0) 303 goto exit; 304 305 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key); 306 if (err < 0) 307 goto exit; 308 309 if (key != mutex->key) { 310 err = NFP_ERRNO(EPERM); 311 goto exit; 312 } 313 314 if (value != MUTEX_LOCKED(interface)) { 315 err = NFP_ERRNO(EACCES); 316 goto exit; 317 } 318 319 err = nfp_cpp_writel(cpp, muw, mutex->address, MUTEX_UNLOCK(interface)); 320 if (err < 0) 321 goto exit; 322 323 mutex->depth = 0; 324 325 exit: 326 return err; 327 } 328 329 /* 330 * Attempt to lock a mutex handle, using the NFP MU Atomic Engine 331 * 332 * Valid lock states: 333 * 334 * 0x....0000 - Unlocked 335 * 0x....000f - Locked 336 * 337 * @param mutex NFP CPP Mutex handle 338 * @return 0 if the lock succeeded, -1 on failure (and errno set 339 * appropriately). 340 */ 341 int 342 nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) 343 { 344 uint32_t mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 345 uint32_t muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 346 uint32_t mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */ 347 uint32_t key, value, tmp; 348 struct nfp_cpp *cpp = mutex->cpp; 349 int err; 350 351 if (mutex->depth > 0) { 352 if (mutex->depth == MUTEX_DEPTH_MAX) 353 return NFP_ERRNO(E2BIG); 354 355 mutex->depth++; 356 return 0; 357 } 358 359 /* Verify that the lock marker is not damaged */ 360 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key); 361 if (err < 0) 362 goto exit; 363 364 if (key != mutex->key) { 365 err = NFP_ERRNO(EPERM); 366 goto exit; 367 } 368 369 /* 370 * Compare against the unlocked state, and if true, 371 * write the interface id into the top 16 bits, and 372 * mark as locked. 373 */ 374 value = MUTEX_LOCKED(nfp_cpp_interface(cpp)); 375 376 /* 377 * We use test_set_imm here, as it implies a read 378 * of the current state, and sets the bits in the 379 * bytemask of the command to 1s. Since the mutex 380 * is guaranteed to be 64-bit aligned, the bytemask 381 * of this 32-bit command is ensured to be 8'b00001111, 382 * which implies that the lower 4 bits will be set to 383 * ones regardless of the initial state. 384 * 385 * Since this is a 'Readback' operation, with no Pull 386 * data, we can treat this as a normal Push (read) 387 * atomic, which returns the original value. 388 */ 389 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp); 390 if (err < 0) 391 goto exit; 392 393 /* Was it unlocked? */ 394 if (MUTEX_IS_UNLOCKED(tmp)) { 395 /* 396 * The read value can only be 0x....0000 in the unlocked state. 397 * If there was another contending for this lock, then 398 * the lock state would be 0x....000f 399 * 400 * Write our owner ID into the lock 401 * While not strictly necessary, this helps with 402 * debug and bookkeeping. 403 */ 404 err = nfp_cpp_writel(cpp, muw, mutex->address, value); 405 if (err < 0) 406 goto exit; 407 408 mutex->depth = 1; 409 goto exit; 410 } 411 412 /* Already locked by us? Success! */ 413 if (tmp == value) { 414 mutex->depth = 1; 415 goto exit; 416 } 417 418 err = NFP_ERRNO(MUTEX_IS_LOCKED(tmp) ? EBUSY : EINVAL); 419 420 exit: 421 return err; 422 } 423