1b843c749SSergey Zigachev /*
2b843c749SSergey Zigachev * Copyright 2008 Advanced Micro Devices, Inc.
3b843c749SSergey Zigachev * Copyright 2008 Red Hat Inc.
4b843c749SSergey Zigachev * Copyright 2009 Jerome Glisse.
5b843c749SSergey Zigachev *
6b843c749SSergey Zigachev * Permission is hereby granted, free of charge, to any person obtaining a
7b843c749SSergey Zigachev * copy of this software and associated documentation files (the "Software"),
8b843c749SSergey Zigachev * to deal in the Software without restriction, including without limitation
9b843c749SSergey Zigachev * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10b843c749SSergey Zigachev * and/or sell copies of the Software, and to permit persons to whom the
11b843c749SSergey Zigachev * Software is furnished to do so, subject to the following conditions:
12b843c749SSergey Zigachev *
13b843c749SSergey Zigachev * The above copyright notice and this permission notice shall be included in
14b843c749SSergey Zigachev * all copies or substantial portions of the Software.
15b843c749SSergey Zigachev *
16b843c749SSergey Zigachev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17b843c749SSergey Zigachev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18b843c749SSergey Zigachev * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19b843c749SSergey Zigachev * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20b843c749SSergey Zigachev * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21b843c749SSergey Zigachev * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22b843c749SSergey Zigachev * OTHER DEALINGS IN THE SOFTWARE.
23b843c749SSergey Zigachev *
24b843c749SSergey Zigachev * Authors: Dave Airlie
25b843c749SSergey Zigachev * Alex Deucher
26b843c749SSergey Zigachev * Jerome Glisse
27b843c749SSergey Zigachev */
28b843c749SSergey Zigachev #include <linux/ktime.h>
29b843c749SSergey Zigachev #include <linux/pagemap.h>
30b843c749SSergey Zigachev #include <drm/drmP.h>
31b843c749SSergey Zigachev #include <drm/amdgpu_drm.h>
32b843c749SSergey Zigachev #include "amdgpu.h"
33b843c749SSergey Zigachev #include "amdgpu_display.h"
34b843c749SSergey Zigachev
amdgpu_gem_object_free(struct drm_gem_object * gobj)35b843c749SSergey Zigachev void amdgpu_gem_object_free(struct drm_gem_object *gobj)
36b843c749SSergey Zigachev {
37b843c749SSergey Zigachev struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj);
38b843c749SSergey Zigachev
39b843c749SSergey Zigachev if (robj) {
40b843c749SSergey Zigachev amdgpu_mn_unregister(robj);
41b843c749SSergey Zigachev amdgpu_bo_unref(&robj);
42b843c749SSergey Zigachev }
43b843c749SSergey Zigachev }
44b843c749SSergey Zigachev
amdgpu_gem_object_create(struct amdgpu_device * adev,unsigned long size,int alignment,u32 initial_domain,u64 flags,enum ttm_bo_type type,struct reservation_object * resv,struct drm_gem_object ** obj)45b843c749SSergey Zigachev int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
46b843c749SSergey Zigachev int alignment, u32 initial_domain,
47b843c749SSergey Zigachev u64 flags, enum ttm_bo_type type,
48b843c749SSergey Zigachev struct reservation_object *resv,
49b843c749SSergey Zigachev struct drm_gem_object **obj)
50b843c749SSergey Zigachev {
51b843c749SSergey Zigachev struct amdgpu_bo *bo;
52b843c749SSergey Zigachev struct amdgpu_bo_param bp;
53b843c749SSergey Zigachev int r;
54b843c749SSergey Zigachev
55b843c749SSergey Zigachev memset(&bp, 0, sizeof(bp));
56b843c749SSergey Zigachev *obj = NULL;
57b843c749SSergey Zigachev /* At least align on page size */
58b843c749SSergey Zigachev if (alignment < PAGE_SIZE) {
59b843c749SSergey Zigachev alignment = PAGE_SIZE;
60b843c749SSergey Zigachev }
61b843c749SSergey Zigachev
62b843c749SSergey Zigachev bp.size = size;
63b843c749SSergey Zigachev bp.byte_align = alignment;
64b843c749SSergey Zigachev bp.type = type;
65b843c749SSergey Zigachev bp.resv = resv;
66b843c749SSergey Zigachev bp.preferred_domain = initial_domain;
67b843c749SSergey Zigachev retry:
68b843c749SSergey Zigachev bp.flags = flags;
69b843c749SSergey Zigachev bp.domain = initial_domain;
70b843c749SSergey Zigachev r = amdgpu_bo_create(adev, &bp, &bo);
71b843c749SSergey Zigachev if (r) {
72b843c749SSergey Zigachev if (r != -ERESTARTSYS) {
73b843c749SSergey Zigachev if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
74b843c749SSergey Zigachev flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
75b843c749SSergey Zigachev goto retry;
76b843c749SSergey Zigachev }
77b843c749SSergey Zigachev
78b843c749SSergey Zigachev if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
79b843c749SSergey Zigachev initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
80b843c749SSergey Zigachev goto retry;
81b843c749SSergey Zigachev }
82b843c749SSergey Zigachev DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
83b843c749SSergey Zigachev size, initial_domain, alignment, r);
84b843c749SSergey Zigachev }
85b843c749SSergey Zigachev return r;
86b843c749SSergey Zigachev }
87b843c749SSergey Zigachev *obj = &bo->gem_base;
88b843c749SSergey Zigachev
89b843c749SSergey Zigachev return 0;
90b843c749SSergey Zigachev }
91b843c749SSergey Zigachev
amdgpu_gem_force_release(struct amdgpu_device * adev)92b843c749SSergey Zigachev void amdgpu_gem_force_release(struct amdgpu_device *adev)
93b843c749SSergey Zigachev {
94b843c749SSergey Zigachev struct drm_device *ddev = adev->ddev;
95b843c749SSergey Zigachev struct drm_file *file;
96b843c749SSergey Zigachev
97b843c749SSergey Zigachev mutex_lock(&ddev->filelist_mutex);
98b843c749SSergey Zigachev
99b843c749SSergey Zigachev list_for_each_entry(file, &ddev->filelist, lhead) {
100b843c749SSergey Zigachev struct drm_gem_object *gobj;
101b843c749SSergey Zigachev int handle;
102b843c749SSergey Zigachev
103b843c749SSergey Zigachev WARN_ONCE(1, "Still active user space clients!\n");
104*78973132SSergey Zigachev lockmgr(&file->table_lock, LK_EXCLUSIVE);
105b843c749SSergey Zigachev idr_for_each_entry(&file->object_idr, gobj, handle) {
106b843c749SSergey Zigachev WARN_ONCE(1, "And also active allocations!\n");
107b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
108b843c749SSergey Zigachev }
109b843c749SSergey Zigachev idr_destroy(&file->object_idr);
110*78973132SSergey Zigachev lockmgr(&file->table_lock, LK_RELEASE);
111b843c749SSergey Zigachev }
112b843c749SSergey Zigachev
113b843c749SSergey Zigachev mutex_unlock(&ddev->filelist_mutex);
114b843c749SSergey Zigachev }
115b843c749SSergey Zigachev
116b843c749SSergey Zigachev /*
117b843c749SSergey Zigachev * Call from drm_gem_handle_create which appear in both new and open ioctl
118b843c749SSergey Zigachev * case.
119b843c749SSergey Zigachev */
amdgpu_gem_object_open(struct drm_gem_object * obj,struct drm_file * file_priv)120b843c749SSergey Zigachev int amdgpu_gem_object_open(struct drm_gem_object *obj,
121b843c749SSergey Zigachev struct drm_file *file_priv)
122b843c749SSergey Zigachev {
123b843c749SSergey Zigachev struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
124b843c749SSergey Zigachev struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
125b843c749SSergey Zigachev struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
126b843c749SSergey Zigachev struct amdgpu_vm *vm = &fpriv->vm;
127b843c749SSergey Zigachev struct amdgpu_bo_va *bo_va;
128b843c749SSergey Zigachev struct mm_struct *mm;
129b843c749SSergey Zigachev int r;
130b843c749SSergey Zigachev
131b843c749SSergey Zigachev mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
132b843c749SSergey Zigachev if (mm && mm != current->mm)
133b843c749SSergey Zigachev return -EPERM;
134b843c749SSergey Zigachev
135b843c749SSergey Zigachev if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
136b843c749SSergey Zigachev abo->tbo.resv != vm->root.base.bo->tbo.resv)
137b843c749SSergey Zigachev return -EPERM;
138b843c749SSergey Zigachev
139b843c749SSergey Zigachev r = amdgpu_bo_reserve(abo, false);
140b843c749SSergey Zigachev if (r)
141b843c749SSergey Zigachev return r;
142b843c749SSergey Zigachev
143b843c749SSergey Zigachev bo_va = amdgpu_vm_bo_find(vm, abo);
144b843c749SSergey Zigachev if (!bo_va) {
145b843c749SSergey Zigachev bo_va = amdgpu_vm_bo_add(adev, vm, abo);
146b843c749SSergey Zigachev } else {
147b843c749SSergey Zigachev ++bo_va->ref_count;
148b843c749SSergey Zigachev }
149b843c749SSergey Zigachev amdgpu_bo_unreserve(abo);
150b843c749SSergey Zigachev return 0;
151b843c749SSergey Zigachev }
152b843c749SSergey Zigachev
amdgpu_gem_object_close(struct drm_gem_object * obj,struct drm_file * file_priv)153b843c749SSergey Zigachev void amdgpu_gem_object_close(struct drm_gem_object *obj,
154b843c749SSergey Zigachev struct drm_file *file_priv)
155b843c749SSergey Zigachev {
156b843c749SSergey Zigachev struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
157b843c749SSergey Zigachev struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
158b843c749SSergey Zigachev struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
159b843c749SSergey Zigachev struct amdgpu_vm *vm = &fpriv->vm;
160b843c749SSergey Zigachev
161b843c749SSergey Zigachev struct amdgpu_bo_list_entry vm_pd;
162b843c749SSergey Zigachev struct list_head list, duplicates;
163b843c749SSergey Zigachev struct ttm_validate_buffer tv;
164b843c749SSergey Zigachev struct ww_acquire_ctx ticket;
165b843c749SSergey Zigachev struct amdgpu_bo_va *bo_va;
166b843c749SSergey Zigachev int r;
167b843c749SSergey Zigachev
168b843c749SSergey Zigachev INIT_LIST_HEAD(&list);
169b843c749SSergey Zigachev INIT_LIST_HEAD(&duplicates);
170b843c749SSergey Zigachev
171b843c749SSergey Zigachev tv.bo = &bo->tbo;
172b843c749SSergey Zigachev tv.shared = true;
173b843c749SSergey Zigachev list_add(&tv.head, &list);
174b843c749SSergey Zigachev
175b843c749SSergey Zigachev amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
176b843c749SSergey Zigachev
177b843c749SSergey Zigachev r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
178b843c749SSergey Zigachev if (r) {
179b843c749SSergey Zigachev dev_err(adev->dev, "leaking bo va because "
180b843c749SSergey Zigachev "we fail to reserve bo (%d)\n", r);
181b843c749SSergey Zigachev return;
182b843c749SSergey Zigachev }
183b843c749SSergey Zigachev bo_va = amdgpu_vm_bo_find(vm, bo);
184b843c749SSergey Zigachev if (bo_va && --bo_va->ref_count == 0) {
185b843c749SSergey Zigachev amdgpu_vm_bo_rmv(adev, bo_va);
186b843c749SSergey Zigachev
187b843c749SSergey Zigachev if (amdgpu_vm_ready(vm)) {
188b843c749SSergey Zigachev struct dma_fence *fence = NULL;
189b843c749SSergey Zigachev
190b843c749SSergey Zigachev r = amdgpu_vm_clear_freed(adev, vm, &fence);
191b843c749SSergey Zigachev if (unlikely(r)) {
192b843c749SSergey Zigachev dev_err(adev->dev, "failed to clear page "
193b843c749SSergey Zigachev "tables on GEM object close (%d)\n", r);
194b843c749SSergey Zigachev }
195b843c749SSergey Zigachev
196b843c749SSergey Zigachev if (fence) {
197b843c749SSergey Zigachev amdgpu_bo_fence(bo, fence, true);
198b843c749SSergey Zigachev dma_fence_put(fence);
199b843c749SSergey Zigachev }
200b843c749SSergey Zigachev }
201b843c749SSergey Zigachev }
202b843c749SSergey Zigachev ttm_eu_backoff_reservation(&ticket, &list);
203b843c749SSergey Zigachev }
204b843c749SSergey Zigachev
205b843c749SSergey Zigachev /*
206b843c749SSergey Zigachev * GEM ioctls.
207b843c749SSergey Zigachev */
amdgpu_gem_create_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)208b843c749SSergey Zigachev int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
209b843c749SSergey Zigachev struct drm_file *filp)
210b843c749SSergey Zigachev {
211b843c749SSergey Zigachev struct amdgpu_device *adev = dev->dev_private;
212b843c749SSergey Zigachev struct amdgpu_fpriv *fpriv = filp->driver_priv;
213b843c749SSergey Zigachev struct amdgpu_vm *vm = &fpriv->vm;
214b843c749SSergey Zigachev union drm_amdgpu_gem_create *args = data;
215b843c749SSergey Zigachev uint64_t flags = args->in.domain_flags;
216b843c749SSergey Zigachev uint64_t size = args->in.bo_size;
217b843c749SSergey Zigachev struct reservation_object *resv = NULL;
218b843c749SSergey Zigachev struct drm_gem_object *gobj;
219b843c749SSergey Zigachev uint32_t handle;
220b843c749SSergey Zigachev int r;
221b843c749SSergey Zigachev
222b843c749SSergey Zigachev /* reject invalid gem flags */
223b843c749SSergey Zigachev if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
224b843c749SSergey Zigachev AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
225b843c749SSergey Zigachev AMDGPU_GEM_CREATE_CPU_GTT_USWC |
226b843c749SSergey Zigachev AMDGPU_GEM_CREATE_VRAM_CLEARED |
227b843c749SSergey Zigachev AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
228b843c749SSergey Zigachev AMDGPU_GEM_CREATE_EXPLICIT_SYNC))
229b843c749SSergey Zigachev
230b843c749SSergey Zigachev return -EINVAL;
231b843c749SSergey Zigachev
232b843c749SSergey Zigachev /* reject invalid gem domains */
233b843c749SSergey Zigachev if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
234b843c749SSergey Zigachev return -EINVAL;
235b843c749SSergey Zigachev
236b843c749SSergey Zigachev /* create a gem object to contain this object in */
237b843c749SSergey Zigachev if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
238b843c749SSergey Zigachev AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
239b843c749SSergey Zigachev if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
240b843c749SSergey Zigachev /* if gds bo is created from user space, it must be
241b843c749SSergey Zigachev * passed to bo list
242b843c749SSergey Zigachev */
243b843c749SSergey Zigachev DRM_ERROR("GDS bo cannot be per-vm-bo\n");
244b843c749SSergey Zigachev return -EINVAL;
245b843c749SSergey Zigachev }
246b843c749SSergey Zigachev flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
247b843c749SSergey Zigachev if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
248b843c749SSergey Zigachev size = size << AMDGPU_GDS_SHIFT;
249b843c749SSergey Zigachev else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
250b843c749SSergey Zigachev size = size << AMDGPU_GWS_SHIFT;
251b843c749SSergey Zigachev else if (args->in.domains == AMDGPU_GEM_DOMAIN_OA)
252b843c749SSergey Zigachev size = size << AMDGPU_OA_SHIFT;
253b843c749SSergey Zigachev else
254b843c749SSergey Zigachev return -EINVAL;
255b843c749SSergey Zigachev }
256b843c749SSergey Zigachev size = roundup(size, PAGE_SIZE);
257b843c749SSergey Zigachev
258b843c749SSergey Zigachev if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
259b843c749SSergey Zigachev r = amdgpu_bo_reserve(vm->root.base.bo, false);
260b843c749SSergey Zigachev if (r)
261b843c749SSergey Zigachev return r;
262b843c749SSergey Zigachev
263b843c749SSergey Zigachev resv = vm->root.base.bo->tbo.resv;
264b843c749SSergey Zigachev }
265b843c749SSergey Zigachev
266b843c749SSergey Zigachev r = amdgpu_gem_object_create(adev, size, args->in.alignment,
267b843c749SSergey Zigachev (u32)(0xffffffff & args->in.domains),
268b843c749SSergey Zigachev flags, ttm_bo_type_device, resv, &gobj);
269b843c749SSergey Zigachev if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
270b843c749SSergey Zigachev if (!r) {
271b843c749SSergey Zigachev struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
272b843c749SSergey Zigachev
273b843c749SSergey Zigachev abo->parent = amdgpu_bo_ref(vm->root.base.bo);
274b843c749SSergey Zigachev }
275b843c749SSergey Zigachev amdgpu_bo_unreserve(vm->root.base.bo);
276b843c749SSergey Zigachev }
277b843c749SSergey Zigachev if (r)
278b843c749SSergey Zigachev return r;
279b843c749SSergey Zigachev
280b843c749SSergey Zigachev r = drm_gem_handle_create(filp, gobj, &handle);
281b843c749SSergey Zigachev /* drop reference from allocate - handle holds it now */
282b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
283b843c749SSergey Zigachev if (r)
284b843c749SSergey Zigachev return r;
285b843c749SSergey Zigachev
286b843c749SSergey Zigachev memset(args, 0, sizeof(*args));
287b843c749SSergey Zigachev args->out.handle = handle;
288b843c749SSergey Zigachev return 0;
289b843c749SSergey Zigachev }
290b843c749SSergey Zigachev
amdgpu_gem_userptr_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)291b843c749SSergey Zigachev int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
292b843c749SSergey Zigachev struct drm_file *filp)
293b843c749SSergey Zigachev {
294*78973132SSergey Zigachev kprintf("amdgpu_gem_userptr_ioctl: not implemented\n");
295*78973132SSergey Zigachev return -EINVAL;
296*78973132SSergey Zigachev #if 0
297b843c749SSergey Zigachev struct ttm_operation_ctx ctx = { true, false };
298b843c749SSergey Zigachev struct amdgpu_device *adev = dev->dev_private;
299b843c749SSergey Zigachev struct drm_amdgpu_gem_userptr *args = data;
300b843c749SSergey Zigachev struct drm_gem_object *gobj;
301b843c749SSergey Zigachev struct amdgpu_bo *bo;
302b843c749SSergey Zigachev uint32_t handle;
303b843c749SSergey Zigachev int r;
304b843c749SSergey Zigachev
305b843c749SSergey Zigachev if (offset_in_page(args->addr | args->size))
306b843c749SSergey Zigachev return -EINVAL;
307b843c749SSergey Zigachev
308b843c749SSergey Zigachev /* reject unknown flag values */
309b843c749SSergey Zigachev if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
310b843c749SSergey Zigachev AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
311b843c749SSergey Zigachev AMDGPU_GEM_USERPTR_REGISTER))
312b843c749SSergey Zigachev return -EINVAL;
313b843c749SSergey Zigachev
314b843c749SSergey Zigachev if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
315b843c749SSergey Zigachev !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
316b843c749SSergey Zigachev
317b843c749SSergey Zigachev /* if we want to write to it we must install a MMU notifier */
318b843c749SSergey Zigachev return -EACCES;
319b843c749SSergey Zigachev }
320b843c749SSergey Zigachev
321b843c749SSergey Zigachev /* create a gem object to contain this object in */
322b843c749SSergey Zigachev r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
323b843c749SSergey Zigachev 0, ttm_bo_type_device, NULL, &gobj);
324b843c749SSergey Zigachev if (r)
325b843c749SSergey Zigachev return r;
326b843c749SSergey Zigachev
327b843c749SSergey Zigachev bo = gem_to_amdgpu_bo(gobj);
328b843c749SSergey Zigachev bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
329b843c749SSergey Zigachev bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
330b843c749SSergey Zigachev r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags);
331b843c749SSergey Zigachev if (r)
332b843c749SSergey Zigachev goto release_object;
333b843c749SSergey Zigachev
334b843c749SSergey Zigachev if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) {
335b843c749SSergey Zigachev r = amdgpu_mn_register(bo, args->addr);
336b843c749SSergey Zigachev if (r)
337b843c749SSergey Zigachev goto release_object;
338b843c749SSergey Zigachev }
339b843c749SSergey Zigachev
340b843c749SSergey Zigachev if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
341b843c749SSergey Zigachev r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
342b843c749SSergey Zigachev bo->tbo.ttm->pages);
343b843c749SSergey Zigachev if (r)
344b843c749SSergey Zigachev goto release_object;
345b843c749SSergey Zigachev
346b843c749SSergey Zigachev r = amdgpu_bo_reserve(bo, true);
347b843c749SSergey Zigachev if (r)
348b843c749SSergey Zigachev goto free_pages;
349b843c749SSergey Zigachev
350b843c749SSergey Zigachev amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
351b843c749SSergey Zigachev r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
352b843c749SSergey Zigachev amdgpu_bo_unreserve(bo);
353b843c749SSergey Zigachev if (r)
354b843c749SSergey Zigachev goto free_pages;
355b843c749SSergey Zigachev }
356b843c749SSergey Zigachev
357b843c749SSergey Zigachev r = drm_gem_handle_create(filp, gobj, &handle);
358b843c749SSergey Zigachev /* drop reference from allocate - handle holds it now */
359b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
360b843c749SSergey Zigachev if (r)
361b843c749SSergey Zigachev return r;
362b843c749SSergey Zigachev
363b843c749SSergey Zigachev args->handle = handle;
364b843c749SSergey Zigachev return 0;
365b843c749SSergey Zigachev
366b843c749SSergey Zigachev free_pages:
367b843c749SSergey Zigachev release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
368b843c749SSergey Zigachev
369b843c749SSergey Zigachev release_object:
370b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
371b843c749SSergey Zigachev
372b843c749SSergey Zigachev return r;
373*78973132SSergey Zigachev #endif
374b843c749SSergey Zigachev }
375b843c749SSergey Zigachev
amdgpu_mode_dumb_mmap(struct drm_file * filp,struct drm_device * dev,uint32_t handle,uint64_t * offset_p)376b843c749SSergey Zigachev int amdgpu_mode_dumb_mmap(struct drm_file *filp,
377b843c749SSergey Zigachev struct drm_device *dev,
378b843c749SSergey Zigachev uint32_t handle, uint64_t *offset_p)
379b843c749SSergey Zigachev {
380b843c749SSergey Zigachev struct drm_gem_object *gobj;
381b843c749SSergey Zigachev struct amdgpu_bo *robj;
382b843c749SSergey Zigachev
383b843c749SSergey Zigachev gobj = drm_gem_object_lookup(filp, handle);
384b843c749SSergey Zigachev if (gobj == NULL) {
385b843c749SSergey Zigachev return -ENOENT;
386b843c749SSergey Zigachev }
387b843c749SSergey Zigachev robj = gem_to_amdgpu_bo(gobj);
388b843c749SSergey Zigachev if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
389b843c749SSergey Zigachev (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
390b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
391b843c749SSergey Zigachev return -EPERM;
392b843c749SSergey Zigachev }
393b843c749SSergey Zigachev *offset_p = amdgpu_bo_mmap_offset(robj);
394b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
395b843c749SSergey Zigachev return 0;
396b843c749SSergey Zigachev }
397b843c749SSergey Zigachev
amdgpu_gem_mmap_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)398b843c749SSergey Zigachev int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
399b843c749SSergey Zigachev struct drm_file *filp)
400b843c749SSergey Zigachev {
401b843c749SSergey Zigachev union drm_amdgpu_gem_mmap *args = data;
402b843c749SSergey Zigachev uint32_t handle = args->in.handle;
403b843c749SSergey Zigachev memset(args, 0, sizeof(*args));
404*78973132SSergey Zigachev return amdgpu_mode_dumb_mmap(filp, dev, handle, (uint64_t *)&args->out.addr_ptr);
405b843c749SSergey Zigachev }
406b843c749SSergey Zigachev
407b843c749SSergey Zigachev /**
408b843c749SSergey Zigachev * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
409b843c749SSergey Zigachev *
410b843c749SSergey Zigachev * @timeout_ns: timeout in ns
411b843c749SSergey Zigachev *
412b843c749SSergey Zigachev * Calculate the timeout in jiffies from an absolute timeout in ns.
413b843c749SSergey Zigachev */
amdgpu_gem_timeout(uint64_t timeout_ns)414b843c749SSergey Zigachev unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
415b843c749SSergey Zigachev {
416b843c749SSergey Zigachev unsigned long timeout_jiffies;
417b843c749SSergey Zigachev ktime_t timeout;
418b843c749SSergey Zigachev
419b843c749SSergey Zigachev /* clamp timeout if it's to large */
420b843c749SSergey Zigachev if (((int64_t)timeout_ns) < 0)
421b843c749SSergey Zigachev return MAX_SCHEDULE_TIMEOUT;
422b843c749SSergey Zigachev
423b843c749SSergey Zigachev timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
424b843c749SSergey Zigachev if (ktime_to_ns(timeout) < 0)
425b843c749SSergey Zigachev return 0;
426b843c749SSergey Zigachev
427b843c749SSergey Zigachev timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
428b843c749SSergey Zigachev /* clamp timeout to avoid unsigned-> signed overflow */
429b843c749SSergey Zigachev if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT )
430b843c749SSergey Zigachev return MAX_SCHEDULE_TIMEOUT - 1;
431b843c749SSergey Zigachev
432b843c749SSergey Zigachev return timeout_jiffies;
433b843c749SSergey Zigachev }
434b843c749SSergey Zigachev
amdgpu_gem_wait_idle_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)435b843c749SSergey Zigachev int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
436b843c749SSergey Zigachev struct drm_file *filp)
437b843c749SSergey Zigachev {
438b843c749SSergey Zigachev union drm_amdgpu_gem_wait_idle *args = data;
439b843c749SSergey Zigachev struct drm_gem_object *gobj;
440b843c749SSergey Zigachev struct amdgpu_bo *robj;
441b843c749SSergey Zigachev uint32_t handle = args->in.handle;
442b843c749SSergey Zigachev unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
443b843c749SSergey Zigachev int r = 0;
444b843c749SSergey Zigachev long ret;
445b843c749SSergey Zigachev
446b843c749SSergey Zigachev gobj = drm_gem_object_lookup(filp, handle);
447b843c749SSergey Zigachev if (gobj == NULL) {
448b843c749SSergey Zigachev return -ENOENT;
449b843c749SSergey Zigachev }
450b843c749SSergey Zigachev robj = gem_to_amdgpu_bo(gobj);
451b843c749SSergey Zigachev ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true,
452b843c749SSergey Zigachev timeout);
453b843c749SSergey Zigachev
454b843c749SSergey Zigachev /* ret == 0 means not signaled,
455b843c749SSergey Zigachev * ret > 0 means signaled
456b843c749SSergey Zigachev * ret < 0 means interrupted before timeout
457b843c749SSergey Zigachev */
458b843c749SSergey Zigachev if (ret >= 0) {
459b843c749SSergey Zigachev memset(args, 0, sizeof(*args));
460b843c749SSergey Zigachev args->out.status = (ret == 0);
461b843c749SSergey Zigachev } else
462b843c749SSergey Zigachev r = ret;
463b843c749SSergey Zigachev
464b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
465b843c749SSergey Zigachev return r;
466b843c749SSergey Zigachev }
467b843c749SSergey Zigachev
amdgpu_gem_metadata_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)468b843c749SSergey Zigachev int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
469b843c749SSergey Zigachev struct drm_file *filp)
470b843c749SSergey Zigachev {
471b843c749SSergey Zigachev struct drm_amdgpu_gem_metadata *args = data;
472b843c749SSergey Zigachev struct drm_gem_object *gobj;
473b843c749SSergey Zigachev struct amdgpu_bo *robj;
474b843c749SSergey Zigachev int r = -1;
475b843c749SSergey Zigachev
476b843c749SSergey Zigachev DRM_DEBUG("%d \n", args->handle);
477b843c749SSergey Zigachev gobj = drm_gem_object_lookup(filp, args->handle);
478b843c749SSergey Zigachev if (gobj == NULL)
479b843c749SSergey Zigachev return -ENOENT;
480b843c749SSergey Zigachev robj = gem_to_amdgpu_bo(gobj);
481b843c749SSergey Zigachev
482b843c749SSergey Zigachev r = amdgpu_bo_reserve(robj, false);
483b843c749SSergey Zigachev if (unlikely(r != 0))
484b843c749SSergey Zigachev goto out;
485b843c749SSergey Zigachev
486b843c749SSergey Zigachev if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
487b843c749SSergey Zigachev amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
488b843c749SSergey Zigachev r = amdgpu_bo_get_metadata(robj, args->data.data,
489b843c749SSergey Zigachev sizeof(args->data.data),
490b843c749SSergey Zigachev &args->data.data_size_bytes,
491*78973132SSergey Zigachev (uint64_t *)&args->data.flags);
492b843c749SSergey Zigachev } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
493b843c749SSergey Zigachev if (args->data.data_size_bytes > sizeof(args->data.data)) {
494b843c749SSergey Zigachev r = -EINVAL;
495b843c749SSergey Zigachev goto unreserve;
496b843c749SSergey Zigachev }
497b843c749SSergey Zigachev r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
498b843c749SSergey Zigachev if (!r)
499b843c749SSergey Zigachev r = amdgpu_bo_set_metadata(robj, args->data.data,
500b843c749SSergey Zigachev args->data.data_size_bytes,
501b843c749SSergey Zigachev args->data.flags);
502b843c749SSergey Zigachev }
503b843c749SSergey Zigachev
504b843c749SSergey Zigachev unreserve:
505b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
506b843c749SSergey Zigachev out:
507b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
508b843c749SSergey Zigachev return r;
509b843c749SSergey Zigachev }
510b843c749SSergey Zigachev
511b843c749SSergey Zigachev /**
512b843c749SSergey Zigachev * amdgpu_gem_va_update_vm -update the bo_va in its VM
513b843c749SSergey Zigachev *
514b843c749SSergey Zigachev * @adev: amdgpu_device pointer
515b843c749SSergey Zigachev * @vm: vm to update
516b843c749SSergey Zigachev * @bo_va: bo_va to update
517b843c749SSergey Zigachev * @operation: map, unmap or clear
518b843c749SSergey Zigachev *
519b843c749SSergey Zigachev * Update the bo_va directly after setting its address. Errors are not
520b843c749SSergey Zigachev * vital here, so they are not reported back to userspace.
521b843c749SSergey Zigachev */
amdgpu_gem_va_update_vm(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_va * bo_va,uint32_t operation)522b843c749SSergey Zigachev static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
523b843c749SSergey Zigachev struct amdgpu_vm *vm,
524b843c749SSergey Zigachev struct amdgpu_bo_va *bo_va,
525b843c749SSergey Zigachev uint32_t operation)
526b843c749SSergey Zigachev {
527b843c749SSergey Zigachev int r;
528b843c749SSergey Zigachev
529b843c749SSergey Zigachev if (!amdgpu_vm_ready(vm))
530b843c749SSergey Zigachev return;
531b843c749SSergey Zigachev
532b843c749SSergey Zigachev r = amdgpu_vm_clear_freed(adev, vm, NULL);
533b843c749SSergey Zigachev if (r)
534b843c749SSergey Zigachev goto error;
535b843c749SSergey Zigachev
536b843c749SSergey Zigachev if (operation == AMDGPU_VA_OP_MAP ||
537b843c749SSergey Zigachev operation == AMDGPU_VA_OP_REPLACE) {
538b843c749SSergey Zigachev r = amdgpu_vm_bo_update(adev, bo_va, false);
539b843c749SSergey Zigachev if (r)
540b843c749SSergey Zigachev goto error;
541b843c749SSergey Zigachev }
542b843c749SSergey Zigachev
543b843c749SSergey Zigachev r = amdgpu_vm_update_directories(adev, vm);
544b843c749SSergey Zigachev
545b843c749SSergey Zigachev error:
546b843c749SSergey Zigachev if (r && r != -ERESTARTSYS)
547b843c749SSergey Zigachev DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
548b843c749SSergey Zigachev }
549b843c749SSergey Zigachev
amdgpu_gem_va_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)550b843c749SSergey Zigachev int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
551b843c749SSergey Zigachev struct drm_file *filp)
552b843c749SSergey Zigachev {
553b843c749SSergey Zigachev const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
554b843c749SSergey Zigachev AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
555b843c749SSergey Zigachev AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK;
556b843c749SSergey Zigachev const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
557b843c749SSergey Zigachev AMDGPU_VM_PAGE_PRT;
558b843c749SSergey Zigachev
559b843c749SSergey Zigachev struct drm_amdgpu_gem_va *args = data;
560b843c749SSergey Zigachev struct drm_gem_object *gobj;
561b843c749SSergey Zigachev struct amdgpu_device *adev = dev->dev_private;
562b843c749SSergey Zigachev struct amdgpu_fpriv *fpriv = filp->driver_priv;
563b843c749SSergey Zigachev struct amdgpu_bo *abo;
564b843c749SSergey Zigachev struct amdgpu_bo_va *bo_va;
565b843c749SSergey Zigachev struct amdgpu_bo_list_entry vm_pd;
566b843c749SSergey Zigachev struct ttm_validate_buffer tv;
567b843c749SSergey Zigachev struct ww_acquire_ctx ticket;
568b843c749SSergey Zigachev struct list_head list, duplicates;
569b843c749SSergey Zigachev uint64_t va_flags;
570b843c749SSergey Zigachev uint64_t vm_size;
571b843c749SSergey Zigachev int r = 0;
572b843c749SSergey Zigachev
573b843c749SSergey Zigachev if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
574b843c749SSergey Zigachev dev_dbg(&dev->pdev->dev,
575b843c749SSergey Zigachev "va_address 0x%LX is in reserved area 0x%LX\n",
576b843c749SSergey Zigachev args->va_address, AMDGPU_VA_RESERVED_SIZE);
577b843c749SSergey Zigachev return -EINVAL;
578b843c749SSergey Zigachev }
579b843c749SSergey Zigachev
580b843c749SSergey Zigachev if (args->va_address >= AMDGPU_VA_HOLE_START &&
581b843c749SSergey Zigachev args->va_address < AMDGPU_VA_HOLE_END) {
582b843c749SSergey Zigachev dev_dbg(&dev->pdev->dev,
583b843c749SSergey Zigachev "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n",
584b843c749SSergey Zigachev args->va_address, AMDGPU_VA_HOLE_START,
585b843c749SSergey Zigachev AMDGPU_VA_HOLE_END);
586b843c749SSergey Zigachev return -EINVAL;
587b843c749SSergey Zigachev }
588b843c749SSergey Zigachev
589b843c749SSergey Zigachev args->va_address &= AMDGPU_VA_HOLE_MASK;
590b843c749SSergey Zigachev
591b843c749SSergey Zigachev vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
592b843c749SSergey Zigachev vm_size -= AMDGPU_VA_RESERVED_SIZE;
593b843c749SSergey Zigachev if (args->va_address + args->map_size > vm_size) {
594b843c749SSergey Zigachev dev_dbg(&dev->pdev->dev,
595*78973132SSergey Zigachev "va_address 0x%llx is in top reserved area 0x%lx\n",
596b843c749SSergey Zigachev args->va_address + args->map_size, vm_size);
597b843c749SSergey Zigachev return -EINVAL;
598b843c749SSergey Zigachev }
599b843c749SSergey Zigachev
600b843c749SSergey Zigachev if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
601b843c749SSergey Zigachev dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n",
602b843c749SSergey Zigachev args->flags);
603b843c749SSergey Zigachev return -EINVAL;
604b843c749SSergey Zigachev }
605b843c749SSergey Zigachev
606b843c749SSergey Zigachev switch (args->operation) {
607b843c749SSergey Zigachev case AMDGPU_VA_OP_MAP:
608b843c749SSergey Zigachev case AMDGPU_VA_OP_UNMAP:
609b843c749SSergey Zigachev case AMDGPU_VA_OP_CLEAR:
610b843c749SSergey Zigachev case AMDGPU_VA_OP_REPLACE:
611b843c749SSergey Zigachev break;
612b843c749SSergey Zigachev default:
613b843c749SSergey Zigachev dev_dbg(&dev->pdev->dev, "unsupported operation %d\n",
614b843c749SSergey Zigachev args->operation);
615b843c749SSergey Zigachev return -EINVAL;
616b843c749SSergey Zigachev }
617b843c749SSergey Zigachev
618b843c749SSergey Zigachev INIT_LIST_HEAD(&list);
619b843c749SSergey Zigachev INIT_LIST_HEAD(&duplicates);
620b843c749SSergey Zigachev if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
621b843c749SSergey Zigachev !(args->flags & AMDGPU_VM_PAGE_PRT)) {
622b843c749SSergey Zigachev gobj = drm_gem_object_lookup(filp, args->handle);
623b843c749SSergey Zigachev if (gobj == NULL)
624b843c749SSergey Zigachev return -ENOENT;
625b843c749SSergey Zigachev abo = gem_to_amdgpu_bo(gobj);
626b843c749SSergey Zigachev tv.bo = &abo->tbo;
627b843c749SSergey Zigachev tv.shared = !!(abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID);
628b843c749SSergey Zigachev list_add(&tv.head, &list);
629b843c749SSergey Zigachev } else {
630b843c749SSergey Zigachev gobj = NULL;
631b843c749SSergey Zigachev abo = NULL;
632b843c749SSergey Zigachev }
633b843c749SSergey Zigachev
634b843c749SSergey Zigachev amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
635b843c749SSergey Zigachev
636b843c749SSergey Zigachev r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
637b843c749SSergey Zigachev if (r)
638b843c749SSergey Zigachev goto error_unref;
639b843c749SSergey Zigachev
640b843c749SSergey Zigachev if (abo) {
641b843c749SSergey Zigachev bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
642b843c749SSergey Zigachev if (!bo_va) {
643b843c749SSergey Zigachev r = -ENOENT;
644b843c749SSergey Zigachev goto error_backoff;
645b843c749SSergey Zigachev }
646b843c749SSergey Zigachev } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
647b843c749SSergey Zigachev bo_va = fpriv->prt_va;
648b843c749SSergey Zigachev } else {
649b843c749SSergey Zigachev bo_va = NULL;
650b843c749SSergey Zigachev }
651b843c749SSergey Zigachev
652b843c749SSergey Zigachev switch (args->operation) {
653b843c749SSergey Zigachev case AMDGPU_VA_OP_MAP:
654b843c749SSergey Zigachev r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
655b843c749SSergey Zigachev args->map_size);
656b843c749SSergey Zigachev if (r)
657b843c749SSergey Zigachev goto error_backoff;
658b843c749SSergey Zigachev
659b843c749SSergey Zigachev va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
660b843c749SSergey Zigachev r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
661b843c749SSergey Zigachev args->offset_in_bo, args->map_size,
662b843c749SSergey Zigachev va_flags);
663b843c749SSergey Zigachev break;
664b843c749SSergey Zigachev case AMDGPU_VA_OP_UNMAP:
665b843c749SSergey Zigachev r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
666b843c749SSergey Zigachev break;
667b843c749SSergey Zigachev
668b843c749SSergey Zigachev case AMDGPU_VA_OP_CLEAR:
669b843c749SSergey Zigachev r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
670b843c749SSergey Zigachev args->va_address,
671b843c749SSergey Zigachev args->map_size);
672b843c749SSergey Zigachev break;
673b843c749SSergey Zigachev case AMDGPU_VA_OP_REPLACE:
674b843c749SSergey Zigachev r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
675b843c749SSergey Zigachev args->map_size);
676b843c749SSergey Zigachev if (r)
677b843c749SSergey Zigachev goto error_backoff;
678b843c749SSergey Zigachev
679b843c749SSergey Zigachev va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
680b843c749SSergey Zigachev r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
681b843c749SSergey Zigachev args->offset_in_bo, args->map_size,
682b843c749SSergey Zigachev va_flags);
683b843c749SSergey Zigachev break;
684b843c749SSergey Zigachev default:
685b843c749SSergey Zigachev break;
686b843c749SSergey Zigachev }
687b843c749SSergey Zigachev if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !amdgpu_vm_debug)
688b843c749SSergey Zigachev amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
689b843c749SSergey Zigachev args->operation);
690b843c749SSergey Zigachev
691b843c749SSergey Zigachev error_backoff:
692b843c749SSergey Zigachev ttm_eu_backoff_reservation(&ticket, &list);
693b843c749SSergey Zigachev
694b843c749SSergey Zigachev error_unref:
695b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
696b843c749SSergey Zigachev return r;
697b843c749SSergey Zigachev }
698b843c749SSergey Zigachev
amdgpu_gem_op_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)699b843c749SSergey Zigachev int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
700b843c749SSergey Zigachev struct drm_file *filp)
701b843c749SSergey Zigachev {
702b843c749SSergey Zigachev struct amdgpu_device *adev = dev->dev_private;
703b843c749SSergey Zigachev struct drm_amdgpu_gem_op *args = data;
704b843c749SSergey Zigachev struct drm_gem_object *gobj;
705b843c749SSergey Zigachev struct amdgpu_bo *robj;
706b843c749SSergey Zigachev int r;
707b843c749SSergey Zigachev
708b843c749SSergey Zigachev gobj = drm_gem_object_lookup(filp, args->handle);
709b843c749SSergey Zigachev if (gobj == NULL) {
710b843c749SSergey Zigachev return -ENOENT;
711b843c749SSergey Zigachev }
712b843c749SSergey Zigachev robj = gem_to_amdgpu_bo(gobj);
713b843c749SSergey Zigachev
714b843c749SSergey Zigachev r = amdgpu_bo_reserve(robj, false);
715b843c749SSergey Zigachev if (unlikely(r))
716b843c749SSergey Zigachev goto out;
717b843c749SSergey Zigachev
718b843c749SSergey Zigachev switch (args->op) {
719b843c749SSergey Zigachev case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
720b843c749SSergey Zigachev struct drm_amdgpu_gem_create_in info;
721b843c749SSergey Zigachev void __user *out = u64_to_user_ptr(args->value);
722b843c749SSergey Zigachev
723b843c749SSergey Zigachev info.bo_size = robj->gem_base.size;
724b843c749SSergey Zigachev info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
725b843c749SSergey Zigachev info.domains = robj->preferred_domains;
726b843c749SSergey Zigachev info.domain_flags = robj->flags;
727b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
728b843c749SSergey Zigachev if (copy_to_user(out, &info, sizeof(info)))
729b843c749SSergey Zigachev r = -EFAULT;
730b843c749SSergey Zigachev break;
731b843c749SSergey Zigachev }
732b843c749SSergey Zigachev case AMDGPU_GEM_OP_SET_PLACEMENT:
733b843c749SSergey Zigachev if (robj->prime_shared_count && (args->value & AMDGPU_GEM_DOMAIN_VRAM)) {
734b843c749SSergey Zigachev r = -EINVAL;
735b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
736b843c749SSergey Zigachev break;
737b843c749SSergey Zigachev }
738b843c749SSergey Zigachev if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
739b843c749SSergey Zigachev r = -EPERM;
740b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
741b843c749SSergey Zigachev break;
742b843c749SSergey Zigachev }
743b843c749SSergey Zigachev robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
744b843c749SSergey Zigachev AMDGPU_GEM_DOMAIN_GTT |
745b843c749SSergey Zigachev AMDGPU_GEM_DOMAIN_CPU);
746b843c749SSergey Zigachev robj->allowed_domains = robj->preferred_domains;
747b843c749SSergey Zigachev if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
748b843c749SSergey Zigachev robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
749b843c749SSergey Zigachev
750b843c749SSergey Zigachev if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
751b843c749SSergey Zigachev amdgpu_vm_bo_invalidate(adev, robj, true);
752b843c749SSergey Zigachev
753b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
754b843c749SSergey Zigachev break;
755b843c749SSergey Zigachev default:
756b843c749SSergey Zigachev amdgpu_bo_unreserve(robj);
757b843c749SSergey Zigachev r = -EINVAL;
758b843c749SSergey Zigachev }
759b843c749SSergey Zigachev
760b843c749SSergey Zigachev out:
761b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
762b843c749SSergey Zigachev return r;
763b843c749SSergey Zigachev }
764b843c749SSergey Zigachev
amdgpu_mode_dumb_create(struct drm_file * file_priv,struct drm_device * dev,struct drm_mode_create_dumb * args)765b843c749SSergey Zigachev int amdgpu_mode_dumb_create(struct drm_file *file_priv,
766b843c749SSergey Zigachev struct drm_device *dev,
767b843c749SSergey Zigachev struct drm_mode_create_dumb *args)
768b843c749SSergey Zigachev {
769b843c749SSergey Zigachev struct amdgpu_device *adev = dev->dev_private;
770b843c749SSergey Zigachev struct drm_gem_object *gobj;
771b843c749SSergey Zigachev uint32_t handle;
772b843c749SSergey Zigachev u32 domain;
773b843c749SSergey Zigachev int r;
774b843c749SSergey Zigachev
775b843c749SSergey Zigachev args->pitch = amdgpu_align_pitch(adev, args->width,
776b843c749SSergey Zigachev DIV_ROUND_UP(args->bpp, 8), 0);
777b843c749SSergey Zigachev args->size = (u64)args->pitch * args->height;
778b843c749SSergey Zigachev args->size = ALIGN(args->size, PAGE_SIZE);
779b843c749SSergey Zigachev domain = amdgpu_bo_get_preferred_pin_domain(adev,
780b843c749SSergey Zigachev amdgpu_display_supported_domains(adev));
781b843c749SSergey Zigachev r = amdgpu_gem_object_create(adev, args->size, 0, domain,
782b843c749SSergey Zigachev AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
783b843c749SSergey Zigachev ttm_bo_type_device, NULL, &gobj);
784b843c749SSergey Zigachev if (r)
785b843c749SSergey Zigachev return -ENOMEM;
786b843c749SSergey Zigachev
787b843c749SSergey Zigachev r = drm_gem_handle_create(file_priv, gobj, &handle);
788b843c749SSergey Zigachev /* drop reference from allocate - handle holds it now */
789b843c749SSergey Zigachev drm_gem_object_put_unlocked(gobj);
790b843c749SSergey Zigachev if (r) {
791b843c749SSergey Zigachev return r;
792b843c749SSergey Zigachev }
793b843c749SSergey Zigachev args->handle = handle;
794b843c749SSergey Zigachev return 0;
795b843c749SSergey Zigachev }
796b843c749SSergey Zigachev
797b843c749SSergey Zigachev #if defined(CONFIG_DEBUG_FS)
798b843c749SSergey Zigachev
799b843c749SSergey Zigachev #define amdgpu_debugfs_gem_bo_print_flag(m, bo, flag) \
800b843c749SSergey Zigachev if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
801b843c749SSergey Zigachev seq_printf((m), " " #flag); \
802b843c749SSergey Zigachev }
803b843c749SSergey Zigachev
amdgpu_debugfs_gem_bo_info(int id,void * ptr,void * data)804b843c749SSergey Zigachev static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
805b843c749SSergey Zigachev {
806b843c749SSergey Zigachev struct drm_gem_object *gobj = ptr;
807b843c749SSergey Zigachev struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
808b843c749SSergey Zigachev struct seq_file *m = data;
809b843c749SSergey Zigachev
810b843c749SSergey Zigachev struct dma_buf_attachment *attachment;
811b843c749SSergey Zigachev struct dma_buf *dma_buf;
812b843c749SSergey Zigachev unsigned domain;
813b843c749SSergey Zigachev const char *placement;
814b843c749SSergey Zigachev unsigned pin_count;
815b843c749SSergey Zigachev
816b843c749SSergey Zigachev domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
817b843c749SSergey Zigachev switch (domain) {
818b843c749SSergey Zigachev case AMDGPU_GEM_DOMAIN_VRAM:
819b843c749SSergey Zigachev placement = "VRAM";
820b843c749SSergey Zigachev break;
821b843c749SSergey Zigachev case AMDGPU_GEM_DOMAIN_GTT:
822b843c749SSergey Zigachev placement = " GTT";
823b843c749SSergey Zigachev break;
824b843c749SSergey Zigachev case AMDGPU_GEM_DOMAIN_CPU:
825b843c749SSergey Zigachev default:
826b843c749SSergey Zigachev placement = " CPU";
827b843c749SSergey Zigachev break;
828b843c749SSergey Zigachev }
829b843c749SSergey Zigachev seq_printf(m, "\t0x%08x: %12ld byte %s",
830b843c749SSergey Zigachev id, amdgpu_bo_size(bo), placement);
831b843c749SSergey Zigachev
832b843c749SSergey Zigachev pin_count = READ_ONCE(bo->pin_count);
833b843c749SSergey Zigachev if (pin_count)
834b843c749SSergey Zigachev seq_printf(m, " pin count %d", pin_count);
835b843c749SSergey Zigachev
836b843c749SSergey Zigachev dma_buf = READ_ONCE(bo->gem_base.dma_buf);
837b843c749SSergey Zigachev attachment = READ_ONCE(bo->gem_base.import_attach);
838b843c749SSergey Zigachev
839b843c749SSergey Zigachev if (attachment)
840b843c749SSergey Zigachev seq_printf(m, " imported from %p", dma_buf);
841b843c749SSergey Zigachev else if (dma_buf)
842b843c749SSergey Zigachev seq_printf(m, " exported as %p", dma_buf);
843b843c749SSergey Zigachev
844b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
845b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, NO_CPU_ACCESS);
846b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_GTT_USWC);
847b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CLEARED);
848b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, SHADOW);
849b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
850b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, VM_ALWAYS_VALID);
851b843c749SSergey Zigachev amdgpu_debugfs_gem_bo_print_flag(m, bo, EXPLICIT_SYNC);
852b843c749SSergey Zigachev
853b843c749SSergey Zigachev seq_printf(m, "\n");
854b843c749SSergey Zigachev
855b843c749SSergey Zigachev return 0;
856b843c749SSergey Zigachev }
857b843c749SSergey Zigachev
amdgpu_debugfs_gem_info(struct seq_file * m,void * data)858b843c749SSergey Zigachev static int amdgpu_debugfs_gem_info(struct seq_file *m, void *data)
859b843c749SSergey Zigachev {
860b843c749SSergey Zigachev struct drm_info_node *node = (struct drm_info_node *)m->private;
861b843c749SSergey Zigachev struct drm_device *dev = node->minor->dev;
862b843c749SSergey Zigachev struct drm_file *file;
863b843c749SSergey Zigachev int r;
864b843c749SSergey Zigachev
865b843c749SSergey Zigachev r = mutex_lock_interruptible(&dev->filelist_mutex);
866b843c749SSergey Zigachev if (r)
867b843c749SSergey Zigachev return r;
868b843c749SSergey Zigachev
869b843c749SSergey Zigachev list_for_each_entry(file, &dev->filelist, lhead) {
870b843c749SSergey Zigachev struct task_struct *task;
871b843c749SSergey Zigachev
872b843c749SSergey Zigachev /*
873b843c749SSergey Zigachev * Although we have a valid reference on file->pid, that does
874b843c749SSergey Zigachev * not guarantee that the task_struct who called get_pid() is
875b843c749SSergey Zigachev * still alive (e.g. get_pid(current) => fork() => exit()).
876b843c749SSergey Zigachev * Therefore, we need to protect this ->comm access using RCU.
877b843c749SSergey Zigachev */
878b843c749SSergey Zigachev rcu_read_lock();
879b843c749SSergey Zigachev task = pid_task(file->pid, PIDTYPE_PID);
880b843c749SSergey Zigachev seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid),
881b843c749SSergey Zigachev task ? task->comm : "<unknown>");
882b843c749SSergey Zigachev rcu_read_unlock();
883b843c749SSergey Zigachev
884*78973132SSergey Zigachev lockmgr(&file->table_lock, LK_EXCLUSIVE);
885b843c749SSergey Zigachev idr_for_each(&file->object_idr, amdgpu_debugfs_gem_bo_info, m);
886*78973132SSergey Zigachev lockmgr(&file->table_lock, LK_RELEASE);
887b843c749SSergey Zigachev }
888b843c749SSergey Zigachev
889b843c749SSergey Zigachev mutex_unlock(&dev->filelist_mutex);
890b843c749SSergey Zigachev return 0;
891b843c749SSergey Zigachev }
892b843c749SSergey Zigachev
893b843c749SSergey Zigachev static const struct drm_info_list amdgpu_debugfs_gem_list[] = {
894b843c749SSergey Zigachev {"amdgpu_gem_info", &amdgpu_debugfs_gem_info, 0, NULL},
895b843c749SSergey Zigachev };
896b843c749SSergey Zigachev #endif
897b843c749SSergey Zigachev
amdgpu_debugfs_gem_init(struct amdgpu_device * adev)898b843c749SSergey Zigachev int amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
899b843c749SSergey Zigachev {
900b843c749SSergey Zigachev #if defined(CONFIG_DEBUG_FS)
901b843c749SSergey Zigachev return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1);
902b843c749SSergey Zigachev #endif
903b843c749SSergey Zigachev return 0;
904b843c749SSergey Zigachev }
905