1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <linux/sched.h> 34 #include <linux/highmem.h> 35 #include <linux/pagemap.h> 36 #include <linux/shmem_fs.h> 37 #include <linux/file.h> 38 #include <linux/swap.h> 39 #include <linux/slab.h> 40 #include <linux/export.h> 41 #include <linux/printk.h> 42 #include <drm/drm_cache.h> 43 #include <drm/drm_mem_util.h> 44 #include <drm/ttm/ttm_module.h> 45 #include <drm/ttm/ttm_bo_driver.h> 46 #include <drm/ttm/ttm_placement.h> 47 #include <drm/ttm/ttm_page_alloc.h> 48 #include <drm/bus_dma_hacks.h> 49 50 /** 51 * Allocates storage for pointers to the pages that back the ttm. 52 */ 53 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 54 { 55 ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); 56 } 57 58 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 59 { 60 ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); 61 #ifndef __NetBSD__ 62 ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, 63 sizeof(*ttm->dma_address)); 64 #endif 65 } 66 67 #ifdef CONFIG_X86 68 static inline int ttm_tt_set_page_caching(struct page *p, 69 enum ttm_caching_state c_old, 70 enum ttm_caching_state c_new) 71 { 72 #ifdef __NetBSD__ 73 return 0; 74 #else 75 int ret = 0; 76 77 if (PageHighMem(p)) 78 return 0; 79 80 if (c_old != tt_cached) { 81 /* p isn't in the default caching state, set it to 82 * writeback first to free its current memtype. */ 83 84 ret = set_pages_wb(p, 1); 85 if (ret) 86 return ret; 87 } 88 89 if (c_new == tt_wc) 90 ret = set_memory_wc((unsigned long) page_address(p), 1); 91 else if (c_new == tt_uncached) 92 ret = set_pages_uc(p, 1); 93 94 return ret; 95 #endif 96 } 97 #else /* CONFIG_X86 */ 98 static inline int ttm_tt_set_page_caching(struct page *p, 99 enum ttm_caching_state c_old, 100 enum ttm_caching_state c_new) 101 { 102 return 0; 103 } 104 #endif /* CONFIG_X86 */ 105 106 /* 107 * Change caching policy for the linear kernel map 108 * for range of pages in a ttm. 109 */ 110 111 static int ttm_tt_set_caching(struct ttm_tt *ttm, 112 enum ttm_caching_state c_state) 113 { 114 int i, j; 115 struct page *cur_page; 116 int ret; 117 118 if (ttm->caching_state == c_state) 119 return 0; 120 121 if (ttm->state == tt_unpopulated) { 122 /* Change caching but don't populate */ 123 ttm->caching_state = c_state; 124 return 0; 125 } 126 127 if (ttm->caching_state == tt_cached) 128 drm_clflush_pages(ttm->pages, ttm->num_pages); 129 130 for (i = 0; i < ttm->num_pages; ++i) { 131 cur_page = ttm->pages[i]; 132 if (likely(cur_page != NULL)) { 133 ret = ttm_tt_set_page_caching(cur_page, 134 ttm->caching_state, 135 c_state); 136 if (unlikely(ret != 0)) 137 goto out_err; 138 } 139 } 140 141 ttm->caching_state = c_state; 142 143 return 0; 144 145 out_err: 146 for (j = 0; j < i; ++j) { 147 cur_page = ttm->pages[j]; 148 if (likely(cur_page != NULL)) { 149 (void)ttm_tt_set_page_caching(cur_page, c_state, 150 ttm->caching_state); 151 } 152 } 153 154 return ret; 155 } 156 157 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 158 { 159 enum ttm_caching_state state; 160 161 if (placement & TTM_PL_FLAG_WC) 162 state = tt_wc; 163 else if (placement & TTM_PL_FLAG_UNCACHED) 164 state = tt_uncached; 165 else 166 state = tt_cached; 167 168 return ttm_tt_set_caching(ttm, state); 169 } 170 EXPORT_SYMBOL(ttm_tt_set_placement_caching); 171 172 void ttm_tt_destroy(struct ttm_tt *ttm) 173 { 174 if (unlikely(ttm == NULL)) 175 return; 176 177 if (ttm->state == tt_bound) { 178 ttm_tt_unbind(ttm); 179 } 180 181 if (ttm->state == tt_unbound) 182 ttm_tt_unpopulate(ttm); 183 184 #ifndef __NetBSD__ 185 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 186 ttm->swap_storage) 187 fput(ttm->swap_storage); 188 189 ttm->swap_storage = NULL; 190 #endif 191 ttm->func->destroy(ttm); 192 } 193 194 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, 195 unsigned long size, uint32_t page_flags, 196 struct page *dummy_read_page) 197 { 198 ttm->bdev = bdev; 199 ttm->glob = bdev->glob; 200 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 201 ttm->caching_state = tt_cached; 202 ttm->page_flags = page_flags; 203 ttm->dummy_read_page = dummy_read_page; 204 ttm->state = tt_unpopulated; 205 #ifdef __NetBSD__ 206 ttm->swap_storage = uao_create(roundup2(size, PAGE_SIZE), 0); 207 uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(bdev->dmat)); 208 #else 209 ttm->swap_storage = NULL; 210 #endif 211 TAILQ_INIT(&ttm->pglist); 212 213 ttm_tt_alloc_page_directory(ttm); 214 if (!ttm->pages) { 215 ttm_tt_destroy(ttm); 216 pr_err("Failed allocating page table\n"); 217 return -ENOMEM; 218 } 219 return 0; 220 } 221 EXPORT_SYMBOL(ttm_tt_init); 222 223 void ttm_tt_fini(struct ttm_tt *ttm) 224 { 225 #ifdef __NetBSD__ 226 uao_detach(ttm->swap_storage); 227 ttm->swap_storage = NULL; 228 #endif 229 drm_free_large(ttm->pages); 230 ttm->pages = NULL; 231 } 232 EXPORT_SYMBOL(ttm_tt_fini); 233 234 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, 235 unsigned long size, uint32_t page_flags, 236 struct page *dummy_read_page) 237 { 238 struct ttm_tt *ttm = &ttm_dma->ttm; 239 240 ttm->bdev = bdev; 241 ttm->glob = bdev->glob; 242 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 243 ttm->caching_state = tt_cached; 244 ttm->page_flags = page_flags; 245 ttm->dummy_read_page = dummy_read_page; 246 ttm->state = tt_unpopulated; 247 #ifdef __NetBSD__ 248 ttm->swap_storage = uao_create(roundup2(size, PAGE_SIZE), 0); 249 uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(bdev->dmat)); 250 #else 251 ttm->swap_storage = NULL; 252 #endif 253 TAILQ_INIT(&ttm->pglist); 254 255 INIT_LIST_HEAD(&ttm_dma->pages_list); 256 ttm_dma_tt_alloc_page_directory(ttm_dma); 257 #ifdef __NetBSD__ 258 { 259 int error; 260 261 if (ttm->num_pages > (SIZE_MAX / 262 MIN(sizeof(ttm_dma->dma_segs[0]), PAGE_SIZE))) { 263 error = ENOMEM; 264 goto fail0; 265 } 266 ttm_dma->dma_segs = kmem_alloc((ttm->num_pages * 267 sizeof(ttm_dma->dma_segs[0])), KM_SLEEP); 268 error = bus_dmamap_create(ttm->bdev->dmat, 269 (ttm->num_pages * PAGE_SIZE), ttm->num_pages, PAGE_SIZE, 0, 270 BUS_DMA_WAITOK, &ttm_dma->dma_address); 271 if (error) 272 goto fail1; 273 274 return 0; 275 276 fail2: __unused 277 bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address); 278 fail1: kmem_free(ttm_dma->dma_segs, (ttm->num_pages * 279 sizeof(ttm_dma->dma_segs[0]))); 280 fail0: KASSERT(error); 281 ttm_tt_destroy(ttm); 282 /* XXX errno NetBSD->Linux */ 283 return -error; 284 } 285 #else 286 if (!ttm->pages || !ttm_dma->dma_address) { 287 ttm_tt_destroy(ttm); 288 pr_err("Failed allocating page table\n"); 289 return -ENOMEM; 290 } 291 return 0; 292 #endif 293 } 294 EXPORT_SYMBOL(ttm_dma_tt_init); 295 296 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 297 { 298 struct ttm_tt *ttm = &ttm_dma->ttm; 299 300 #ifdef __NetBSD__ 301 uao_detach(ttm->swap_storage); 302 ttm->swap_storage = NULL; 303 #endif 304 drm_free_large(ttm->pages); 305 ttm->pages = NULL; 306 #ifdef __NetBSD__ 307 bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address); 308 kmem_free(ttm_dma->dma_segs, (ttm->num_pages * 309 sizeof(ttm_dma->dma_segs[0]))); 310 #else 311 drm_free_large(ttm_dma->dma_address); 312 ttm_dma->dma_address = NULL; 313 #endif 314 } 315 EXPORT_SYMBOL(ttm_dma_tt_fini); 316 317 void ttm_tt_unbind(struct ttm_tt *ttm) 318 { 319 int ret __diagused; 320 321 if (ttm->state == tt_bound) { 322 ret = ttm->func->unbind(ttm); 323 BUG_ON(ret); 324 ttm->state = tt_unbound; 325 } 326 } 327 328 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 329 { 330 int ret = 0; 331 332 if (!ttm) 333 return -EINVAL; 334 335 if (ttm->state == tt_bound) 336 return 0; 337 338 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 339 if (ret) 340 return ret; 341 342 ret = ttm->func->bind(ttm, bo_mem); 343 if (unlikely(ret != 0)) 344 return ret; 345 346 ttm->state = tt_bound; 347 348 return 0; 349 } 350 EXPORT_SYMBOL(ttm_tt_bind); 351 352 int ttm_tt_swapin(struct ttm_tt *ttm) 353 { 354 #ifdef __NetBSD__ 355 struct uvm_object *uobj = ttm->swap_storage; 356 struct vm_page *page; 357 unsigned i; 358 int error; 359 360 KASSERT(uobj != NULL); 361 error = uvm_obj_wirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT), 362 &ttm->pglist); 363 if (error) 364 /* XXX errno NetBSD->Linux */ 365 return -error; 366 367 i = 0; 368 TAILQ_FOREACH(page, &ttm->pglist, pageq.queue) { 369 KASSERT(i < ttm->num_pages); 370 KASSERT(ttm->pages[i] == NULL); 371 ttm->pages[i] = container_of(page, struct page, p_vmp); 372 i++; 373 } 374 KASSERT(i == ttm->num_pages); 375 376 /* Success! */ 377 return 0; 378 #else 379 struct address_space *swap_space; 380 struct file *swap_storage; 381 struct page *from_page; 382 struct page *to_page; 383 int i; 384 int ret = -ENOMEM; 385 386 swap_storage = ttm->swap_storage; 387 BUG_ON(swap_storage == NULL); 388 389 swap_space = file_inode(swap_storage)->i_mapping; 390 391 for (i = 0; i < ttm->num_pages; ++i) { 392 from_page = shmem_read_mapping_page(swap_space, i); 393 if (IS_ERR(from_page)) { 394 ret = PTR_ERR(from_page); 395 goto out_err; 396 } 397 to_page = ttm->pages[i]; 398 if (unlikely(to_page == NULL)) 399 goto out_err; 400 401 copy_highpage(to_page, from_page); 402 page_cache_release(from_page); 403 } 404 405 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 406 fput(swap_storage); 407 ttm->swap_storage = NULL; 408 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 409 410 return 0; 411 out_err: 412 return ret; 413 #endif 414 } 415 416 #ifdef __NetBSD__ 417 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) 418 { 419 struct uvm_object *uobj = ttm->swap_storage; 420 unsigned i; 421 422 KASSERT((ttm->state == tt_unbound) || (ttm->state == tt_unpopulated)); 423 KASSERT(ttm->caching_state == tt_cached); 424 KASSERT(uobj != NULL); 425 426 /* 427 * XXX Dunno what this persistent swap storage business is all 428 * about, but I see nothing using it and it doesn't make sense. 429 */ 430 KASSERT(persistent_swap_storage == NULL); 431 432 uvm_obj_unwirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT)); 433 for (i = 0; i < ttm->num_pages; i++) 434 ttm->pages[i] = NULL; 435 436 /* Success! */ 437 return 0; 438 } 439 #else 440 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) 441 { 442 struct address_space *swap_space; 443 struct file *swap_storage; 444 struct page *from_page; 445 struct page *to_page; 446 int i; 447 int ret = -ENOMEM; 448 449 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); 450 BUG_ON(ttm->caching_state != tt_cached); 451 452 if (!persistent_swap_storage) { 453 swap_storage = shmem_file_setup("ttm swap", 454 ttm->num_pages << PAGE_SHIFT, 455 0); 456 if (unlikely(IS_ERR(swap_storage))) { 457 pr_err("Failed allocating swap storage\n"); 458 return PTR_ERR(swap_storage); 459 } 460 } else 461 swap_storage = persistent_swap_storage; 462 463 swap_space = file_inode(swap_storage)->i_mapping; 464 465 for (i = 0; i < ttm->num_pages; ++i) { 466 from_page = ttm->pages[i]; 467 if (unlikely(from_page == NULL)) 468 continue; 469 to_page = shmem_read_mapping_page(swap_space, i); 470 if (unlikely(IS_ERR(to_page))) { 471 ret = PTR_ERR(to_page); 472 goto out_err; 473 } 474 copy_highpage(to_page, from_page); 475 set_page_dirty(to_page); 476 mark_page_accessed(to_page); 477 page_cache_release(to_page); 478 } 479 480 ttm_tt_unpopulate(ttm); 481 ttm->swap_storage = swap_storage; 482 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 483 if (persistent_swap_storage) 484 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 485 486 return 0; 487 out_err: 488 if (!persistent_swap_storage) 489 fput(swap_storage); 490 491 return ret; 492 } 493 #endif 494 495 static void ttm_tt_clear_mapping(struct ttm_tt *ttm) 496 { 497 #ifndef __NetBSD__ 498 pgoff_t i; 499 struct page **page = ttm->pages; 500 501 if (ttm->page_flags & TTM_PAGE_FLAG_SG) 502 return; 503 504 for (i = 0; i < ttm->num_pages; ++i) { 505 (*page)->mapping = NULL; 506 (*page++)->index = 0; 507 } 508 #endif 509 } 510 511 void ttm_tt_unpopulate(struct ttm_tt *ttm) 512 { 513 if (ttm->state == tt_unpopulated) 514 return; 515 516 ttm_tt_clear_mapping(ttm); 517 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 518 } 519