1 /* $NetBSD: amdgpu_amdkfd_fence.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $ */
2
3 /*
4 * Copyright 2016-2018 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: amdgpu_amdkfd_fence.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $");
27
28 #include <linux/dma-fence.h>
29 #include <linux/spinlock.h>
30 #include <linux/atomic.h>
31 #include <linux/stacktrace.h>
32 #include <linux/sched.h>
33 #include <linux/slab.h>
34 #include <linux/sched/mm.h>
35 #include "amdgpu_amdkfd.h"
36
37 static const struct dma_fence_ops amdkfd_fence_ops;
38 static atomic_t fence_seq = ATOMIC_INIT(0);
39
40 /* Eviction Fence
41 * Fence helper functions to deal with KFD memory eviction.
42 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
43 * evicted unless all the user queues for that process are evicted.
44 *
45 * All the BOs in a process share an eviction fence. When process X wants
46 * to map VRAM memory but TTM can't find enough space, TTM will attempt to
47 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
48 * by calling ttm_bo_driver->eviction_valuable().
49 *
50 * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
51 * to process X. Otherwise, it will return true to indicate BO can be
52 * evicted by TTM.
53 *
54 * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
55 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
56 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
57 *
58 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
59 * nofity when the BO is free to move. fence_add_callback --> enable_signaling
60 * --> amdgpu_amdkfd_fence.enable_signaling
61 *
62 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
63 * user queues and signal fence. The work item will also start another delayed
64 * work item to restore BOs
65 */
66
amdgpu_amdkfd_fence_create(u64 context,struct mm_struct * mm)67 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
68 struct mm_struct *mm)
69 {
70 struct amdgpu_amdkfd_fence *fence;
71
72 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
73 if (fence == NULL)
74 return NULL;
75
76 /* This reference gets released in amdkfd_fence_release */
77 mmgrab(mm);
78 fence->mm = mm;
79 get_task_comm(fence->timeline_name, current);
80 spin_lock_init(&fence->lock);
81
82 dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
83 context, atomic_inc_return(&fence_seq));
84
85 return fence;
86 }
87
to_amdgpu_amdkfd_fence(struct dma_fence * f)88 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
89 {
90 struct amdgpu_amdkfd_fence *fence;
91
92 if (!f)
93 return NULL;
94
95 fence = container_of(f, struct amdgpu_amdkfd_fence, base);
96 if (fence && f->ops == &amdkfd_fence_ops)
97 return fence;
98
99 return NULL;
100 }
101
amdkfd_fence_get_driver_name(struct dma_fence * f)102 static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
103 {
104 return "amdgpu_amdkfd_fence";
105 }
106
amdkfd_fence_get_timeline_name(struct dma_fence * f)107 static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
108 {
109 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
110
111 return fence->timeline_name;
112 }
113
114 /**
115 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
116 * a KFD BO and schedules a job to move the BO.
117 * If fence is already signaled return true.
118 * If fence is not signaled schedule a evict KFD process work item.
119 */
amdkfd_fence_enable_signaling(struct dma_fence * f)120 static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
121 {
122 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
123
124 if (!fence)
125 return false;
126
127 if (dma_fence_is_signaled(f))
128 return true;
129
130 if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
131 return true;
132
133 return false;
134 }
135
136 /**
137 * amdkfd_fence_release - callback that fence can be freed
138 *
139 * @fence: fence
140 *
141 * This function is called when the reference count becomes zero.
142 * Drops the mm_struct reference and RCU schedules freeing up the fence.
143 */
amdkfd_fence_release(struct dma_fence * f)144 static void amdkfd_fence_release(struct dma_fence *f)
145 {
146 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
147
148 /* Unconditionally signal the fence. The process is getting
149 * terminated.
150 */
151 if (WARN_ON(!fence))
152 return; /* Not an amdgpu_amdkfd_fence */
153
154 mmdrop(fence->mm);
155 kfree_rcu(f, rcu);
156 }
157
158 /**
159 * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
160 * if same return TRUE else return FALSE.
161 *
162 * @f: [IN] fence
163 * @mm: [IN] mm that needs to be verified
164 */
amdkfd_fence_check_mm(struct dma_fence * f,struct mm_struct * mm)165 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
166 {
167 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
168
169 if (!fence)
170 return false;
171 else if (fence->mm == mm)
172 return true;
173
174 return false;
175 }
176
177 static const struct dma_fence_ops amdkfd_fence_ops = {
178 .get_driver_name = amdkfd_fence_get_driver_name,
179 .get_timeline_name = amdkfd_fence_get_timeline_name,
180 .enable_signaling = amdkfd_fence_enable_signaling,
181 .release = amdkfd_fence_release,
182 };
183