1 /* $NetBSD: i915_globals.c,v 1.3 2021/12/19 11:49:11 riastradh Exp $ */
2
3 /*
4 * SPDX-License-Identifier: MIT
5 *
6 * Copyright © 2019 Intel Corporation
7 */
8
9 #include <sys/cdefs.h>
10 __KERNEL_RCSID(0, "$NetBSD: i915_globals.c,v 1.3 2021/12/19 11:49:11 riastradh Exp $");
11
12 #include <linux/slab.h>
13 #include <linux/workqueue.h>
14
15 #include "i915_active.h"
16 #include "gem/i915_gem_context.h"
17 #include "gem/i915_gem_object.h"
18 #include "i915_globals.h"
19 #include "i915_request.h"
20 #include "i915_scheduler.h"
21 #include "i915_vma.h"
22
23 #include <linux/nbsd-namespace.h>
24
25 static LIST_HEAD(globals);
26
27 static atomic_t active;
28 static atomic_t epoch;
29 static struct park_work {
30 struct delayed_work work;
31 struct rcu_head rcu;
32 unsigned long flags;
33 #define PENDING 0
34 int epoch;
35 } park;
36
i915_globals_shrink(void)37 static void i915_globals_shrink(void)
38 {
39 struct i915_global *global;
40
41 /*
42 * kmem_cache_shrink() discards empty slabs and reorders partially
43 * filled slabs to prioritise allocating from the mostly full slabs,
44 * with the aim of reducing fragmentation.
45 */
46 list_for_each_entry(global, &globals, link)
47 global->shrink();
48 }
49
__i915_globals_grace(struct rcu_head * rcu)50 static void __i915_globals_grace(struct rcu_head *rcu)
51 {
52 /* Ratelimit parking as shrinking is quite slow */
53 schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
54 }
55
__i915_globals_queue_rcu(void)56 static void __i915_globals_queue_rcu(void)
57 {
58 park.epoch = atomic_inc_return(&epoch);
59 if (!atomic_read(&active)) {
60 init_rcu_head(&park.rcu);
61 call_rcu(&park.rcu, __i915_globals_grace);
62 }
63 }
64
__i915_globals_park(struct work_struct * work)65 static void __i915_globals_park(struct work_struct *work)
66 {
67 destroy_rcu_head(&park.rcu);
68
69 /* Confirm nothing woke up in the last grace period */
70 if (park.epoch != atomic_read(&epoch)) {
71 __i915_globals_queue_rcu();
72 return;
73 }
74
75 clear_bit(PENDING, &park.flags);
76 i915_globals_shrink();
77 }
78
i915_global_register(struct i915_global * global)79 void __init i915_global_register(struct i915_global *global)
80 {
81 GEM_BUG_ON(!global->shrink);
82 GEM_BUG_ON(!global->exit);
83
84 list_add_tail(&global->link, &globals);
85 }
86
__i915_globals_cleanup(void)87 static void __i915_globals_cleanup(void)
88 {
89 struct i915_global *global, *next;
90
91 list_for_each_entry_safe_reverse(global, next, &globals, link)
92 global->exit();
93 }
94
95 static __initconst int (* const initfn[])(void) = {
96 i915_global_active_init,
97 i915_global_buddy_init,
98 i915_global_context_init,
99 i915_global_gem_context_init,
100 i915_global_objects_init,
101 i915_global_request_init,
102 i915_global_scheduler_init,
103 i915_global_vma_init,
104 };
105
i915_globals_init(void)106 int __init i915_globals_init(void)
107 {
108 int i;
109
110 for (i = 0; i < ARRAY_SIZE(initfn); i++) {
111 int err;
112
113 err = initfn[i]();
114 if (err) {
115 __i915_globals_cleanup();
116 return err;
117 }
118 }
119
120 INIT_DELAYED_WORK(&park.work, __i915_globals_park);
121 return 0;
122 }
123
i915_globals_park(void)124 void i915_globals_park(void)
125 {
126 /*
127 * Defer shrinking the global slab caches (and other work) until
128 * after a RCU grace period has completed with no activity. This
129 * is to try and reduce the latency impact on the consumers caused
130 * by us shrinking the caches the same time as they are trying to
131 * allocate, with the assumption being that if we idle long enough
132 * for an RCU grace period to elapse since the last use, it is likely
133 * to be longer until we need the caches again.
134 */
135 if (!atomic_dec_and_test(&active))
136 return;
137
138 /* Queue cleanup after the next RCU grace period has freed slabs */
139 if (!test_and_set_bit(PENDING, &park.flags))
140 __i915_globals_queue_rcu();
141 }
142
i915_globals_unpark(void)143 void i915_globals_unpark(void)
144 {
145 atomic_inc(&epoch);
146 atomic_inc(&active);
147 }
148
__i915_globals_flush(void)149 static void __exit __i915_globals_flush(void)
150 {
151 atomic_inc(&active); /* skip shrinking */
152
153 rcu_barrier(); /* wait for the work to be queued */
154 flush_delayed_work(&park.work);
155
156 atomic_dec(&active);
157 }
158
i915_globals_exit(void)159 void __exit i915_globals_exit(void)
160 {
161 GEM_BUG_ON(atomic_read(&active));
162
163 __i915_globals_flush();
164 __i915_globals_cleanup();
165
166 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
167 rcu_barrier();
168 }
169