1 /* $NetBSD: vmwgfx_page_dirty.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $ */
2
3 // SPDX-License-Identifier: GPL-2.0 OR MIT
4 /**************************************************************************
5 *
6 * Copyright 2019 VMware, Inc., Palo Alto, CA., USA
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: vmwgfx_page_dirty.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $");
31
32 #include "vmwgfx_drv.h"
33
34 /*
35 * Different methods for tracking dirty:
36 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
37 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
38 * accesses in the VM mkwrite() callback
39 */
40 enum vmw_bo_dirty_method {
41 VMW_BO_DIRTY_PAGETABLE,
42 VMW_BO_DIRTY_MKWRITE,
43 };
44
45 /*
46 * No dirtied pages at scan trigger a transition to the _MKWRITE method,
47 * similarly a certain percentage of dirty pages trigger a transition to
48 * the _PAGETABLE method. How many triggers should we wait for before
49 * changing method?
50 */
51 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
52
53 /* Percentage to trigger a transition to the _PAGETABLE method */
54 #define VMW_DIRTY_PERCENTAGE 10
55
56 /**
57 * struct vmw_bo_dirty - Dirty information for buffer objects
58 * @start: First currently dirty bit
59 * @end: Last currently dirty bit + 1
60 * @method: The currently used dirty method
61 * @change_count: Number of consecutive method change triggers
62 * @ref_count: Reference count for this structure
63 * @bitmap_size: The size of the bitmap in bits. Typically equal to the
64 * nuber of pages in the bo.
65 * @size: The accounting size for this struct.
66 * @bitmap: A bitmap where each bit represents a page. A set bit means a
67 * dirty page.
68 */
69 struct vmw_bo_dirty {
70 unsigned long start;
71 unsigned long end;
72 enum vmw_bo_dirty_method method;
73 unsigned int change_count;
74 unsigned int ref_count;
75 unsigned long bitmap_size;
76 size_t size;
77 unsigned long bitmap[0];
78 };
79
80 /**
81 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
82 * @vbo: The buffer object to scan
83 *
84 * Scans the pagetable for dirty bits. Clear those bits and modify the
85 * dirty structure with the results. This function may change the
86 * dirty-tracking method.
87 */
vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object * vbo)88 static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo)
89 {
90 struct vmw_bo_dirty *dirty = vbo->dirty;
91 pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
92 struct address_space *mapping = vbo->base.bdev->dev_mapping;
93 pgoff_t num_marked;
94
95 num_marked = clean_record_shared_mapping_range
96 (mapping,
97 offset, dirty->bitmap_size,
98 offset, &dirty->bitmap[0],
99 &dirty->start, &dirty->end);
100 if (num_marked == 0)
101 dirty->change_count++;
102 else
103 dirty->change_count = 0;
104
105 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
106 dirty->change_count = 0;
107 dirty->method = VMW_BO_DIRTY_MKWRITE;
108 wp_shared_mapping_range(mapping,
109 offset, dirty->bitmap_size);
110 clean_record_shared_mapping_range(mapping,
111 offset, dirty->bitmap_size,
112 offset, &dirty->bitmap[0],
113 &dirty->start, &dirty->end);
114 }
115 }
116
117 /**
118 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
119 * @vbo: The buffer object to scan
120 *
121 * Write-protect pages written to so that consecutive write accesses will
122 * trigger a call to mkwrite.
123 *
124 * This function may change the dirty-tracking method.
125 */
vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object * vbo)126 static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo)
127 {
128 struct vmw_bo_dirty *dirty = vbo->dirty;
129 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
130 struct address_space *mapping = vbo->base.bdev->dev_mapping;
131 pgoff_t num_marked;
132
133 if (dirty->end <= dirty->start)
134 return;
135
136 num_marked = wp_shared_mapping_range(vbo->base.bdev->dev_mapping,
137 dirty->start + offset,
138 dirty->end - dirty->start);
139
140 if (100UL * num_marked / dirty->bitmap_size >
141 VMW_DIRTY_PERCENTAGE) {
142 dirty->change_count++;
143 } else {
144 dirty->change_count = 0;
145 }
146
147 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
148 pgoff_t start = 0;
149 pgoff_t end = dirty->bitmap_size;
150
151 dirty->method = VMW_BO_DIRTY_PAGETABLE;
152 clean_record_shared_mapping_range(mapping, offset, end, offset,
153 &dirty->bitmap[0],
154 &start, &end);
155 bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
156 if (dirty->start < dirty->end)
157 bitmap_set(&dirty->bitmap[0], dirty->start,
158 dirty->end - dirty->start);
159 dirty->change_count = 0;
160 }
161 }
162
163 /**
164 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
165 * tracking structure
166 * @vbo: The buffer object to scan
167 *
168 * This function may change the dirty tracking method.
169 */
vmw_bo_dirty_scan(struct vmw_buffer_object * vbo)170 void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo)
171 {
172 struct vmw_bo_dirty *dirty = vbo->dirty;
173
174 if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
175 vmw_bo_dirty_scan_pagetable(vbo);
176 else
177 vmw_bo_dirty_scan_mkwrite(vbo);
178 }
179
180 /**
181 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
182 * an unmap_mapping_range operation.
183 * @vbo: The buffer object,
184 * @start: First page of the range within the buffer object.
185 * @end: Last page of the range within the buffer object + 1.
186 *
187 * If we're using the _PAGETABLE scan method, we may leak dirty pages
188 * when calling unmap_mapping_range(). This function makes sure we pick
189 * up all dirty pages.
190 */
vmw_bo_dirty_pre_unmap(struct vmw_buffer_object * vbo,pgoff_t start,pgoff_t end)191 static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo,
192 pgoff_t start, pgoff_t end)
193 {
194 struct vmw_bo_dirty *dirty = vbo->dirty;
195 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
196 struct address_space *mapping = vbo->base.bdev->dev_mapping;
197
198 if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
199 return;
200
201 wp_shared_mapping_range(mapping, start + offset, end - start);
202 clean_record_shared_mapping_range(mapping, start + offset,
203 end - start, offset,
204 &dirty->bitmap[0], &dirty->start,
205 &dirty->end);
206 }
207
208 /**
209 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
210 * @vbo: The buffer object,
211 * @start: First page of the range within the buffer object.
212 * @end: Last page of the range within the buffer object + 1.
213 *
214 * This is similar to ttm_bo_unmap_virtual_locked() except it takes a subrange.
215 */
vmw_bo_dirty_unmap(struct vmw_buffer_object * vbo,pgoff_t start,pgoff_t end)216 void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
217 pgoff_t start, pgoff_t end)
218 {
219 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
220 struct address_space *mapping = vbo->base.bdev->dev_mapping;
221
222 vmw_bo_dirty_pre_unmap(vbo, start, end);
223 unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
224 (loff_t) (end - start) << PAGE_SHIFT);
225 }
226
227 /**
228 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
229 * @vbo: The buffer object
230 *
231 * This function registers a dirty-tracking user to a buffer object.
232 * A user can be for example a resource or a vma in a special user-space
233 * mapping.
234 *
235 * Return: Zero on success, -ENOMEM on memory allocation failure.
236 */
vmw_bo_dirty_add(struct vmw_buffer_object * vbo)237 int vmw_bo_dirty_add(struct vmw_buffer_object *vbo)
238 {
239 struct vmw_bo_dirty *dirty = vbo->dirty;
240 pgoff_t num_pages = vbo->base.num_pages;
241 size_t size, acc_size;
242 int ret;
243 static struct ttm_operation_ctx ctx = {
244 .interruptible = false,
245 .no_wait_gpu = false
246 };
247
248 if (dirty) {
249 dirty->ref_count++;
250 return 0;
251 }
252
253 size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
254 acc_size = ttm_round_pot(size);
255 ret = ttm_mem_global_alloc(&ttm_mem_glob, acc_size, &ctx);
256 if (ret) {
257 VMW_DEBUG_USER("Out of graphics memory for buffer object "
258 "dirty tracker.\n");
259 return ret;
260 }
261 dirty = kvzalloc(size, GFP_KERNEL);
262 if (!dirty) {
263 ret = -ENOMEM;
264 goto out_no_dirty;
265 }
266
267 dirty->size = acc_size;
268 dirty->bitmap_size = num_pages;
269 dirty->start = dirty->bitmap_size;
270 dirty->end = 0;
271 dirty->ref_count = 1;
272 if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
273 dirty->method = VMW_BO_DIRTY_PAGETABLE;
274 } else {
275 struct address_space *mapping = vbo->base.bdev->dev_mapping;
276 pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
277
278 dirty->method = VMW_BO_DIRTY_MKWRITE;
279
280 /* Write-protect and then pick up already dirty bits */
281 wp_shared_mapping_range(mapping, offset, num_pages);
282 clean_record_shared_mapping_range(mapping, offset, num_pages,
283 offset,
284 &dirty->bitmap[0],
285 &dirty->start, &dirty->end);
286 }
287
288 vbo->dirty = dirty;
289
290 return 0;
291
292 out_no_dirty:
293 ttm_mem_global_free(&ttm_mem_glob, acc_size);
294 return ret;
295 }
296
297 /**
298 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
299 * @vbo: The buffer object
300 *
301 * This function releases a dirty-tracking user from a buffer object.
302 * If the reference count reaches zero, then the dirty-tracking object is
303 * freed and the pointer to it cleared.
304 *
305 * Return: Zero on success, -ENOMEM on memory allocation failure.
306 */
vmw_bo_dirty_release(struct vmw_buffer_object * vbo)307 void vmw_bo_dirty_release(struct vmw_buffer_object *vbo)
308 {
309 struct vmw_bo_dirty *dirty = vbo->dirty;
310
311 if (dirty && --dirty->ref_count == 0) {
312 size_t acc_size = dirty->size;
313
314 kvfree(dirty);
315 ttm_mem_global_free(&ttm_mem_glob, acc_size);
316 vbo->dirty = NULL;
317 }
318 }
319
320 /**
321 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
322 * its backing mob.
323 * @res: The resource
324 *
325 * This function will pick up all dirty ranges affecting the resource from
326 * it's backup mob, and call vmw_resource_dirty_update() once for each
327 * range. The transferred ranges will be cleared from the backing mob's
328 * dirty tracking.
329 */
vmw_bo_dirty_transfer_to_res(struct vmw_resource * res)330 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
331 {
332 struct vmw_buffer_object *vbo = res->backup;
333 struct vmw_bo_dirty *dirty = vbo->dirty;
334 pgoff_t start, cur, end;
335 unsigned long res_start = res->backup_offset;
336 unsigned long res_end = res->backup_offset + res->backup_size;
337
338 WARN_ON_ONCE(res_start & ~PAGE_MASK);
339 res_start >>= PAGE_SHIFT;
340 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
341
342 if (res_start >= dirty->end || res_end <= dirty->start)
343 return;
344
345 cur = max(res_start, dirty->start);
346 res_end = max(res_end, dirty->end);
347 while (cur < res_end) {
348 unsigned long num;
349
350 start = find_next_bit(&dirty->bitmap[0], res_end, cur);
351 if (start >= res_end)
352 break;
353
354 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
355 cur = end + 1;
356 num = end - start;
357 bitmap_clear(&dirty->bitmap[0], start, num);
358 vmw_resource_dirty_update(res, start, end);
359 }
360
361 if (res_start <= dirty->start && res_end > dirty->start)
362 dirty->start = res_end;
363 if (res_start < dirty->end && res_end >= dirty->end)
364 dirty->end = res_start;
365 }
366
367 /**
368 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
369 * its backing mob.
370 * @res: The resource
371 *
372 * This function will clear all dirty ranges affecting the resource from
373 * it's backup mob's dirty tracking.
374 */
vmw_bo_dirty_clear_res(struct vmw_resource * res)375 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
376 {
377 unsigned long res_start = res->backup_offset;
378 unsigned long res_end = res->backup_offset + res->backup_size;
379 struct vmw_buffer_object *vbo = res->backup;
380 struct vmw_bo_dirty *dirty = vbo->dirty;
381
382 res_start >>= PAGE_SHIFT;
383 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
384
385 if (res_start >= dirty->end || res_end <= dirty->start)
386 return;
387
388 res_start = max(res_start, dirty->start);
389 res_end = min(res_end, dirty->end);
390 bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
391
392 if (res_start <= dirty->start && res_end > dirty->start)
393 dirty->start = res_end;
394 if (res_start < dirty->end && res_end >= dirty->end)
395 dirty->end = res_start;
396 }
397
vmw_bo_vm_mkwrite(struct vm_fault * vmf)398 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
399 {
400 struct vm_area_struct *vma = vmf->vma;
401 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
402 vma->vm_private_data;
403 vm_fault_t ret;
404 unsigned long page_offset;
405 unsigned int save_flags;
406 struct vmw_buffer_object *vbo =
407 container_of(bo, typeof(*vbo), base);
408
409 /*
410 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
411 * So make sure the TTM helpers are aware.
412 */
413 save_flags = vmf->flags;
414 vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
415 ret = ttm_bo_vm_reserve(bo, vmf);
416 vmf->flags = save_flags;
417 if (ret)
418 return ret;
419
420 page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
421 if (unlikely(page_offset >= bo->num_pages)) {
422 ret = VM_FAULT_SIGBUS;
423 goto out_unlock;
424 }
425
426 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
427 !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
428 struct vmw_bo_dirty *dirty = vbo->dirty;
429
430 __set_bit(page_offset, &dirty->bitmap[0]);
431 dirty->start = min(dirty->start, page_offset);
432 dirty->end = max(dirty->end, page_offset + 1);
433 }
434
435 out_unlock:
436 dma_resv_unlock(bo->base.resv);
437 return ret;
438 }
439
vmw_bo_vm_fault(struct vm_fault * vmf)440 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
441 {
442 struct vm_area_struct *vma = vmf->vma;
443 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
444 vma->vm_private_data;
445 struct vmw_buffer_object *vbo =
446 container_of(bo, struct vmw_buffer_object, base);
447 pgoff_t num_prefault;
448 pgprot_t prot;
449 vm_fault_t ret;
450
451 ret = ttm_bo_vm_reserve(bo, vmf);
452 if (ret)
453 return ret;
454
455 num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
456 TTM_BO_VM_NUM_PREFAULT;
457
458 if (vbo->dirty) {
459 pgoff_t allowed_prefault;
460 unsigned long page_offset;
461
462 page_offset = vmf->pgoff -
463 drm_vma_node_start(&bo->base.vma_node);
464 if (page_offset >= bo->num_pages ||
465 vmw_resources_clean(vbo, page_offset,
466 page_offset + PAGE_SIZE,
467 &allowed_prefault)) {
468 ret = VM_FAULT_SIGBUS;
469 goto out_unlock;
470 }
471
472 num_prefault = min(num_prefault, allowed_prefault);
473 }
474
475 /*
476 * If we don't track dirty using the MKWRITE method, make sure
477 * sure the page protection is write-enabled so we don't get
478 * a lot of unnecessary write faults.
479 */
480 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
481 prot = vma->vm_page_prot;
482 else
483 prot = vm_get_page_prot(vma->vm_flags);
484
485 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
486 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
487 return ret;
488
489 out_unlock:
490 dma_resv_unlock(bo->base.resv);
491
492 return ret;
493 }
494