1 /************************************************************************** 2 * 3 * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 /** @file ttm_ref_object.c 31 * 32 * Base- and reference object implementation for the various 33 * ttm objects. Implements reference counting, minimal security checks 34 * and release on file close. 35 */ 36 37 /** 38 * struct ttm_object_file 39 * 40 * @tdev: Pointer to the ttm_object_device. 41 * 42 * @lock: Lock that protects the ref_list list and the 43 * ref_hash hash tables. 44 * 45 * @ref_list: List of ttm_ref_objects to be destroyed at 46 * file release. 47 * 48 * @ref_hash: Hash tables of ref objects, one per ttm_ref_type, 49 * for fast lookup of ref objects given a base object. 50 */ 51 52 #define pr_fmt(fmt) "[TTM] " fmt 53 54 #include <drm/drmP.h> 55 #include <drm/ttm/ttm_object.h> 56 #include <drm/ttm/ttm_module.h> 57 #include <linux/export.h> 58 59 struct ttm_object_file { 60 struct ttm_object_device *tdev; 61 struct lock lock; 62 struct list_head ref_list; 63 struct drm_open_hash ref_hash[TTM_REF_NUM]; 64 struct kref refcount; 65 }; 66 67 /** 68 * struct ttm_object_device 69 * 70 * @object_lock: lock that protects the object_hash hash table. 71 * 72 * @object_hash: hash table for fast lookup of object global names. 73 * 74 * @object_count: Per device object count. 75 * 76 * This is the per-device data structure needed for ttm object management. 77 */ 78 79 struct ttm_object_device { 80 struct lock object_lock; 81 struct drm_open_hash object_hash; 82 atomic_t object_count; 83 struct ttm_mem_global *mem_glob; 84 }; 85 86 /** 87 * struct ttm_ref_object 88 * 89 * @hash: Hash entry for the per-file object reference hash. 90 * 91 * @head: List entry for the per-file list of ref-objects. 92 * 93 * @kref: Ref count. 94 * 95 * @obj: Base object this ref object is referencing. 96 * 97 * @ref_type: Type of ref object. 98 * 99 * This is similar to an idr object, but it also has a hash table entry 100 * that allows lookup with a pointer to the referenced object as a key. In 101 * that way, one can easily detect whether a base object is referenced by 102 * a particular ttm_object_file. It also carries a ref count to avoid creating 103 * multiple ref objects if a ttm_object_file references the same base 104 * object more than once. 105 */ 106 107 struct ttm_ref_object { 108 struct drm_hash_item hash; 109 struct list_head head; 110 struct kref kref; 111 enum ttm_ref_type ref_type; 112 struct ttm_base_object *obj; 113 struct ttm_object_file *tfile; 114 }; 115 116 MALLOC_DEFINE(M_TTM_OBJ_FILE, "ttm_obj_file", "TTM File Objects"); 117 118 static inline struct ttm_object_file * 119 ttm_object_file_ref(struct ttm_object_file *tfile) 120 { 121 kref_get(&tfile->refcount); 122 return tfile; 123 } 124 125 static void ttm_object_file_destroy(struct kref *kref) 126 { 127 struct ttm_object_file *tfile = 128 container_of(kref, struct ttm_object_file, refcount); 129 130 drm_free(tfile, M_TTM_OBJ_FILE); 131 } 132 133 134 static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile) 135 { 136 struct ttm_object_file *tfile = *p_tfile; 137 138 *p_tfile = NULL; 139 kref_put(&tfile->refcount, ttm_object_file_destroy); 140 } 141 142 143 int ttm_base_object_init(struct ttm_object_file *tfile, 144 struct ttm_base_object *base, 145 bool shareable, 146 enum ttm_object_type object_type, 147 void (*refcount_release) (struct ttm_base_object **), 148 void (*ref_obj_release) (struct ttm_base_object *, 149 enum ttm_ref_type ref_type)) 150 { 151 struct ttm_object_device *tdev = tfile->tdev; 152 int ret; 153 154 base->shareable = shareable; 155 base->tfile = ttm_object_file_ref(tfile); 156 base->refcount_release = refcount_release; 157 base->ref_obj_release = ref_obj_release; 158 base->object_type = object_type; 159 kref_init(&base->refcount); 160 lockinit(&tdev->object_lock, "ttmbao", 0, LK_CANRECURSE); 161 lockmgr(&tdev->object_lock, LK_EXCLUSIVE); 162 ret = drm_ht_just_insert_please(&tdev->object_hash, 163 &base->hash, 164 (unsigned long)base, 31, 0, 0); 165 lockmgr(&tdev->object_lock, LK_RELEASE); 166 if (unlikely(ret != 0)) 167 goto out_err0; 168 169 ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); 170 if (unlikely(ret != 0)) 171 goto out_err1; 172 173 ttm_base_object_unref(&base); 174 175 return 0; 176 out_err1: 177 lockmgr(&tdev->object_lock, LK_EXCLUSIVE); 178 (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); 179 lockmgr(&tdev->object_lock, LK_RELEASE); 180 out_err0: 181 return ret; 182 } 183 EXPORT_SYMBOL(ttm_base_object_init); 184 185 static void ttm_release_base(struct kref *kref) 186 { 187 struct ttm_base_object *base = 188 container_of(kref, struct ttm_base_object, refcount); 189 struct ttm_object_device *tdev = base->tfile->tdev; 190 191 if (atomic_read(&kref->refcount)) { 192 lockmgr(&tdev->object_lock, LK_RELEASE); 193 return; 194 } 195 (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); 196 lockmgr(&tdev->object_lock, LK_RELEASE); 197 198 /* 199 * Note: We don't use synchronize_rcu() here because it's far 200 * too slow. It's up to the user to free the object using 201 * call_rcu() or ttm_base_object_kfree(). 202 */ 203 204 if (base->refcount_release) { 205 ttm_object_file_unref(&base->tfile); 206 base->refcount_release(&base); 207 } 208 } 209 210 void ttm_base_object_unref(struct ttm_base_object **p_base) 211 { 212 struct ttm_base_object *base = *p_base; 213 struct ttm_object_device *tdev = base->tfile->tdev; 214 215 *p_base = NULL; 216 217 lockmgr(&tdev->object_lock, LK_EXCLUSIVE); 218 if (kref_put(&base->refcount, ttm_release_base) == 0) { 219 lockmgr(&tdev->object_lock, LK_RELEASE); 220 } 221 } 222 EXPORT_SYMBOL(ttm_base_object_unref); 223 224 struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, 225 uint32_t key) 226 { 227 struct ttm_object_device *tdev = tfile->tdev; 228 struct ttm_base_object *base; 229 struct drm_hash_item *hash; 230 int ret; 231 232 lockmgr(&tdev->object_lock, LK_EXCLUSIVE); 233 ret = drm_ht_find_item(&tdev->object_hash, key, &hash); 234 235 if (likely(ret == 0)) { 236 base = drm_hash_entry(hash, struct ttm_base_object, hash); 237 ret = kref_get_unless_zero(&base->refcount) ? 0 : -EINVAL; 238 } 239 lockmgr(&tdev->object_lock, LK_RELEASE); 240 241 if (unlikely(ret != 0)) 242 return NULL; 243 244 if (tfile != base->tfile && !base->shareable) { 245 kprintf("[TTM] Attempted access of non-shareable object %p\n", 246 base); 247 ttm_base_object_unref(&base); 248 return NULL; 249 } 250 251 return base; 252 } 253 EXPORT_SYMBOL(ttm_base_object_lookup); 254 255 MALLOC_DEFINE(M_TTM_OBJ_REF, "ttm_obj_ref", "TTM Ref Objects"); 256 257 int ttm_ref_object_add(struct ttm_object_file *tfile, 258 struct ttm_base_object *base, 259 enum ttm_ref_type ref_type, bool *existed) 260 { 261 struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; 262 struct ttm_ref_object *ref; 263 struct drm_hash_item *hash; 264 struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; 265 int ret = -EINVAL; 266 267 if (existed != NULL) 268 *existed = true; 269 270 while (ret == -EINVAL) { 271 lockmgr(&tfile->lock, LK_EXCLUSIVE); 272 ret = drm_ht_find_item(ht, base->hash.key, &hash); 273 274 if (ret == 0) { 275 ref = drm_hash_entry(hash, struct ttm_ref_object, hash); 276 kref_get(&ref->kref); 277 lockmgr(&tfile->lock, LK_RELEASE); 278 break; 279 } 280 281 lockmgr(&tfile->lock, LK_RELEASE); 282 ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), 283 false, false); 284 if (unlikely(ret != 0)) 285 return ret; 286 ref = kmalloc(sizeof(*ref), M_TTM_OBJ_REF, M_WAITOK); 287 if (unlikely(ref == NULL)) { 288 ttm_mem_global_free(mem_glob, sizeof(*ref)); 289 return -ENOMEM; 290 } 291 292 ref->hash.key = base->hash.key; 293 ref->obj = base; 294 ref->tfile = tfile; 295 ref->ref_type = ref_type; 296 kref_init(&ref->kref); 297 298 lockmgr(&tfile->lock, LK_EXCLUSIVE); 299 ret = drm_ht_insert_item(ht, &ref->hash); 300 301 if (likely(ret == 0)) { 302 list_add_tail(&ref->head, &tfile->ref_list); 303 kref_get(&base->refcount); 304 lockmgr(&tfile->lock, LK_RELEASE); 305 if (existed != NULL) 306 *existed = false; 307 break; 308 } 309 310 lockmgr(&tfile->lock, LK_RELEASE); 311 BUG_ON(ret != -EINVAL); 312 313 ttm_mem_global_free(mem_glob, sizeof(*ref)); 314 drm_free(ref, M_TTM_OBJ_REF); 315 } 316 317 return ret; 318 } 319 EXPORT_SYMBOL(ttm_ref_object_add); 320 321 static void ttm_ref_object_release(struct kref *kref) 322 { 323 struct ttm_ref_object *ref = 324 container_of(kref, struct ttm_ref_object, kref); 325 struct ttm_base_object *base = ref->obj; 326 struct ttm_object_file *tfile = ref->tfile; 327 struct drm_open_hash *ht; 328 struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; 329 330 ht = &tfile->ref_hash[ref->ref_type]; 331 (void)drm_ht_remove_item(ht, &ref->hash); 332 list_del(&ref->head); 333 lockmgr(&tfile->lock, LK_RELEASE); 334 335 if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) 336 base->ref_obj_release(base, ref->ref_type); 337 338 ttm_base_object_unref(&ref->obj); 339 ttm_mem_global_free(mem_glob, sizeof(*ref)); 340 drm_free(ref, M_TTM_OBJ_REF); 341 lockmgr(&tfile->lock, LK_EXCLUSIVE); 342 } 343 344 int ttm_ref_object_base_unref(struct ttm_object_file *tfile, 345 unsigned long key, enum ttm_ref_type ref_type) 346 { 347 struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; 348 struct ttm_ref_object *ref; 349 struct drm_hash_item *hash; 350 int ret; 351 352 lockmgr(&tfile->lock, LK_EXCLUSIVE); 353 ret = drm_ht_find_item(ht, key, &hash); 354 if (unlikely(ret != 0)) { 355 lockmgr(&tfile->lock, LK_RELEASE); 356 return -EINVAL; 357 } 358 ref = drm_hash_entry(hash, struct ttm_ref_object, hash); 359 kref_put(&ref->kref, ttm_ref_object_release); 360 lockmgr(&tfile->lock, LK_RELEASE); 361 return 0; 362 } 363 EXPORT_SYMBOL(ttm_ref_object_base_unref); 364 365 void ttm_object_file_release(struct ttm_object_file **p_tfile) 366 { 367 struct ttm_ref_object *ref; 368 struct list_head *list; 369 unsigned int i; 370 struct ttm_object_file *tfile = *p_tfile; 371 372 *p_tfile = NULL; 373 lockmgr(&tfile->lock, LK_EXCLUSIVE); 374 375 /* 376 * Since we release the lock within the loop, we have to 377 * restart it from the beginning each time. 378 */ 379 380 while (!list_empty(&tfile->ref_list)) { 381 list = tfile->ref_list.next; 382 ref = list_entry(list, struct ttm_ref_object, head); 383 ttm_ref_object_release(&ref->kref); 384 } 385 386 for (i = 0; i < TTM_REF_NUM; ++i) 387 drm_ht_remove(&tfile->ref_hash[i]); 388 389 lockmgr(&tfile->lock, LK_RELEASE); 390 ttm_object_file_unref(&tfile); 391 } 392 EXPORT_SYMBOL(ttm_object_file_release); 393 394 struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, 395 unsigned int hash_order) 396 { 397 struct ttm_object_file *tfile; 398 unsigned int i; 399 unsigned int j = 0; 400 int ret; 401 402 tfile = kmalloc(sizeof(*tfile), M_TTM_OBJ_FILE, M_WAITOK); 403 if (unlikely(tfile == NULL)) 404 return NULL; 405 406 lockinit(&tfile->lock, "ttmfo", 0, LK_CANRECURSE); 407 tfile->tdev = tdev; 408 kref_init(&tfile->refcount); 409 INIT_LIST_HEAD(&tfile->ref_list); 410 411 for (i = 0; i < TTM_REF_NUM; ++i) { 412 ret = drm_ht_create(&tfile->ref_hash[i], hash_order); 413 if (ret) { 414 j = i; 415 goto out_err; 416 } 417 } 418 419 return tfile; 420 out_err: 421 for (i = 0; i < j; ++i) 422 drm_ht_remove(&tfile->ref_hash[i]); 423 424 drm_free(tfile, M_TTM_OBJ_FILE); 425 426 return NULL; 427 } 428 EXPORT_SYMBOL(ttm_object_file_init); 429 430 MALLOC_DEFINE(M_TTM_OBJ_DEV, "ttm_obj_dev", "TTM Device Objects"); 431 432 struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global 433 *mem_glob, 434 unsigned int hash_order) 435 { 436 struct ttm_object_device *tdev; 437 int ret; 438 439 tdev = kmalloc(sizeof(*tdev), M_TTM_OBJ_DEV, M_WAITOK); 440 if (unlikely(tdev == NULL)) 441 return NULL; 442 443 tdev->mem_glob = mem_glob; 444 lockinit(&tdev->object_lock, "ttmdo", 0, LK_CANRECURSE); 445 atomic_set(&tdev->object_count, 0); 446 ret = drm_ht_create(&tdev->object_hash, hash_order); 447 448 if (likely(ret == 0)) 449 return tdev; 450 451 drm_free(tdev, M_TTM_OBJ_DEV); 452 return NULL; 453 } 454 EXPORT_SYMBOL(ttm_object_device_init); 455 456 void ttm_object_device_release(struct ttm_object_device **p_tdev) 457 { 458 struct ttm_object_device *tdev = *p_tdev; 459 460 *p_tdev = NULL; 461 462 lockmgr(&tdev->object_lock, LK_EXCLUSIVE); 463 drm_ht_remove(&tdev->object_hash); 464 lockmgr(&tdev->object_lock, LK_RELEASE); 465 466 drm_free(tdev, M_TTM_OBJ_DEV); 467 } 468 EXPORT_SYMBOL(ttm_object_device_release); 469