1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Netronome Systems, Inc. 3 * All rights reserved. 4 */ 5 6 #include "nfp_mutex.h" 7 8 #include <sched.h> 9 10 #include "nfp_logs.h" 11 #include "nfp_target.h" 12 13 /* 14 * If you need more than 65536 recursive locks, please 15 * rethink your code. 16 */ 17 #define MUTEX_DEPTH_MAX 0xffff 18 19 struct nfp_cpp_mutex { 20 struct nfp_cpp *cpp; 21 uint8_t target; 22 uint16_t depth; 23 uint64_t address; 24 uint32_t key; 25 uint32_t usage; 26 struct nfp_cpp_mutex *prev, *next; 27 }; 28 29 static inline uint32_t 30 nfp_mutex_locked(uint16_t interface) 31 { 32 return (uint32_t)interface << 16 | 0x000f; 33 } 34 35 static inline uint32_t 36 nfp_mutex_unlocked(uint16_t interface) 37 { 38 return (uint32_t)interface << 16 | 0x0000; 39 } 40 41 static inline uint16_t 42 nfp_mutex_owner(uint32_t val) 43 { 44 return (val >> 16) & 0xffff; 45 } 46 47 static inline bool 48 nfp_mutex_is_locked(uint32_t val) 49 { 50 return (val & 0xffff) == 0x000f; 51 } 52 53 static inline bool 54 nfp_mutex_is_unlocked(uint32_t val) 55 { 56 return (val & 0xffff) == 0; 57 } 58 59 static int 60 nfp_cpp_mutex_validate(uint16_t interface, 61 int *target, 62 uint64_t address) 63 { 64 /* Not permitted on invalid interfaces */ 65 if (NFP_CPP_INTERFACE_TYPE_of(interface) == NFP_CPP_INTERFACE_TYPE_INVALID) 66 return -EINVAL; 67 68 /* Address must be 64-bit aligned */ 69 if ((address & 7) != 0) 70 return -EINVAL; 71 72 if (*target != NFP_CPP_TARGET_MU) 73 return -EINVAL; 74 75 return 0; 76 } 77 78 /** 79 * Initialize a mutex location 80 * 81 * The CPP target:address must point to a 64-bit aligned location, and 82 * will initialize 64 bits of data at the location. 83 * 84 * This creates the initial mutex state, as locked by this 85 * nfp_cpp_interface(). 86 * 87 * This function should only be called when setting up 88 * the initial lock state upon boot-up of the system. 89 * 90 * @param cpp 91 * NFP CPP handle 92 * @param target 93 * NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 94 * @param address 95 * Offset into the address space of the NFP CPP target ID 96 * @param key 97 * Unique 32-bit value for this mutex 98 * 99 * @return 100 * 0 on success, or negative value on failure 101 */ 102 int 103 nfp_cpp_mutex_init(struct nfp_cpp *cpp, 104 int target, 105 uint64_t address, 106 uint32_t key) 107 { 108 int err; 109 uint32_t muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 110 uint16_t interface = nfp_cpp_interface(cpp); 111 112 err = nfp_cpp_mutex_validate(interface, &target, address); 113 if (err < 0) 114 return err; 115 116 err = nfp_cpp_writel(cpp, muw, address + 4, key); 117 if (err < 0) 118 return err; 119 120 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface)); 121 if (err < 0) 122 return err; 123 124 return 0; 125 } 126 127 /** 128 * Create a mutex handle from an address controlled by a MU Atomic engine 129 * 130 * The CPP target:address must point to a 64-bit aligned location, and 131 * reserve 64 bits of data at the location for use by the handle. 132 * 133 * Only target/address pairs that point to entities that support the 134 * MU Atomic Engine are supported. 135 * 136 * @param cpp 137 * NFP CPP handle 138 * @param target 139 * NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 140 * @param address 141 * Offset into the address space of the NFP CPP target ID 142 * @param key 143 * 32-bit unique key (must match the key at this location) 144 * 145 * @return 146 * A non-NULL struct nfp_cpp_mutex * on success, NULL on failure. 147 */ 148 struct nfp_cpp_mutex * 149 nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, 150 int target, 151 uint64_t address, 152 uint32_t key) 153 { 154 int err; 155 uint32_t tmp; 156 struct nfp_cpp_mutex *mutex; 157 uint32_t mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 158 uint16_t interface = nfp_cpp_interface(cpp); 159 160 err = nfp_cpp_mutex_validate(interface, &target, address); 161 if (err < 0) 162 return NULL; 163 164 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp); 165 if (err < 0) 166 return NULL; 167 168 if (tmp != key) 169 return NULL; 170 171 mutex = calloc(1, sizeof(*mutex)); 172 if (mutex == NULL) 173 return NULL; 174 175 mutex->cpp = cpp; 176 mutex->target = target; 177 mutex->address = address; 178 mutex->key = key; 179 mutex->depth = 0; 180 181 return mutex; 182 } 183 184 /** 185 * Free a mutex handle - does not alter the lock state 186 * 187 * @param mutex 188 * NFP CPP Mutex handle 189 */ 190 void 191 nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex) 192 { 193 free(mutex); 194 } 195 196 /** 197 * Lock a mutex handle, using the NFP MU Atomic Engine 198 * 199 * @param mutex 200 * NFP CPP Mutex handle 201 * 202 * @return 203 * 0 on success, or negative value on failure. 204 */ 205 int 206 nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex) 207 { 208 int err; 209 time_t warn_at = time(NULL) + 15; 210 211 while ((err = nfp_cpp_mutex_trylock(mutex)) != 0) { 212 /* If err != -EBUSY, then the lock was damaged */ 213 if (err < 0 && err != -EBUSY) 214 return err; 215 216 if (time(NULL) >= warn_at) { 217 PMD_DRV_LOG(WARNING, "Waiting for NFP mutex..."); 218 warn_at = time(NULL) + 60; 219 } 220 221 sched_yield(); 222 } 223 224 return 0; 225 } 226 227 /** 228 * Unlock a mutex handle, using the NFP MU Atomic Engine 229 * 230 * @param mutex 231 * NFP CPP Mutex handle 232 * 233 * @return 234 * 0 on success, or negative value on failure 235 */ 236 int 237 nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex) 238 { 239 int err; 240 uint32_t key; 241 uint32_t value; 242 struct nfp_cpp *cpp = mutex->cpp; 243 uint16_t interface = nfp_cpp_interface(cpp); 244 uint32_t muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 245 uint32_t mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 246 247 if (mutex->depth > 1) { 248 mutex->depth--; 249 return 0; 250 } 251 252 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key); 253 if (err < 0) 254 return err; 255 256 if (key != mutex->key) 257 return -EPERM; 258 259 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value); 260 if (err < 0) 261 return err; 262 263 if (value != nfp_mutex_locked(interface)) 264 return -EACCES; 265 266 err = nfp_cpp_writel(cpp, muw, mutex->address, 267 nfp_mutex_unlocked(interface)); 268 if (err < 0) 269 return err; 270 271 mutex->depth = 0; 272 273 return 0; 274 } 275 276 /** 277 * Attempt to lock a mutex handle, using the NFP MU Atomic Engine 278 * 279 * Valid lock states: 280 * 0x....0000 - Unlocked 281 * 0x....000f - Locked 282 * 283 * @param mutex 284 * NFP CPP Mutex handle 285 * 286 * @return 287 * 0 if the lock succeeded, negative value on failure. 288 */ 289 int 290 nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) 291 { 292 int err; 293 uint32_t key; 294 uint32_t tmp; 295 uint32_t value; 296 struct nfp_cpp *cpp = mutex->cpp; 297 uint32_t mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 298 uint32_t muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 299 uint32_t mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */ 300 301 if (mutex->depth > 0) { 302 if (mutex->depth == MUTEX_DEPTH_MAX) 303 return -E2BIG; 304 305 mutex->depth++; 306 return 0; 307 } 308 309 /* Verify that the lock marker is not damaged */ 310 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key); 311 if (err < 0) { 312 PMD_DRV_LOG(ERR, "Failed to read key."); 313 return err; 314 } 315 316 if (key != mutex->key) { 317 PMD_DRV_LOG(ERR, "Key: %x is not same with the mutex: %x.", 318 key, mutex->key); 319 return -EPERM; 320 } 321 322 /* 323 * Compare against the unlocked state, and if true, 324 * write the interface id into the top 16 bits, and 325 * mark as locked. 326 */ 327 value = nfp_mutex_locked(nfp_cpp_interface(cpp)); 328 329 /* 330 * We use test_set_imm here, as it implies a read 331 * of the current state, and sets the bits in the 332 * bytemask of the command to 1s. Since the mutex 333 * is guaranteed to be 64-bit aligned, the bytemask 334 * of this 32-bit command is ensured to be 8'b00001111, 335 * which implies that the lower 4 bits will be set to 336 * ones regardless of the initial state. 337 * 338 * Since this is a 'Readback' operation, with no Pull 339 * data, we can treat this as a normal Push (read) 340 * atomic, which returns the original value. 341 */ 342 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp); 343 if (err < 0) { 344 PMD_DRV_LOG(ERR, "Failed to read tmp."); 345 return err; 346 } 347 348 /* Was it unlocked? */ 349 if (nfp_mutex_is_unlocked(tmp)) { 350 /* 351 * The read value can only be 0x....0000 in the unlocked state. 352 * If there was another contending for this lock, then 353 * the lock state would be 0x....000f 354 * 355 * Write our owner ID into the lock. 356 * While not strictly necessary, this helps with 357 * debug and bookkeeping. 358 */ 359 err = nfp_cpp_writel(cpp, muw, mutex->address, value); 360 if (err < 0) { 361 PMD_DRV_LOG(ERR, "Failed to write value."); 362 return err; 363 } 364 365 mutex->depth = 1; 366 return 0; 367 } 368 369 /* Already locked by us? Success! */ 370 if (tmp == value) { 371 mutex->depth = 1; 372 return 0; 373 } 374 375 return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL; 376 } 377 378 /** 379 * Release lock if held by local system. 380 * Extreme care is advised, call only when no local lock users can exist. 381 * 382 * @param cpp 383 * NFP CPP handle 384 * @param target 385 * NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 386 * @param address 387 * Offset into the address space of the NFP CPP target ID 388 * 389 * @return 390 * - (0) if the lock was OK 391 * - (1) if locked by us 392 * - (-errno) on invalid mutex 393 */ 394 int 395 nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, 396 int target, 397 uint64_t address) 398 { 399 int err; 400 uint32_t tmp; 401 uint16_t interface = nfp_cpp_interface(cpp); 402 const uint32_t mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 403 const uint32_t muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 404 405 err = nfp_cpp_mutex_validate(interface, &target, address); 406 if (err != 0) 407 return err; 408 409 /* Check lock */ 410 err = nfp_cpp_readl(cpp, mur, address, &tmp); 411 if (err < 0) 412 return err; 413 414 if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface) 415 return 0; 416 417 /* Bust the lock */ 418 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface)); 419 if (err < 0) 420 return err; 421 422 return 1; 423 } 424