xref: /freebsd-src/sys/compat/linuxkpi/common/src/linux_rcu.c (revision 1a01b4e566c36b9ca697d27429306fe961036101)
1*1a01b4e5SHans Petter Selasky /*-
2*1a01b4e5SHans Petter Selasky  * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org)
3*1a01b4e5SHans Petter Selasky  * All rights reserved.
4*1a01b4e5SHans Petter Selasky  *
5*1a01b4e5SHans Petter Selasky  * Redistribution and use in source and binary forms, with or without
6*1a01b4e5SHans Petter Selasky  * modification, are permitted provided that the following conditions
7*1a01b4e5SHans Petter Selasky  * are met:
8*1a01b4e5SHans Petter Selasky  * 1. Redistributions of source code must retain the above copyright
9*1a01b4e5SHans Petter Selasky  *    notice unmodified, this list of conditions, and the following
10*1a01b4e5SHans Petter Selasky  *    disclaimer.
11*1a01b4e5SHans Petter Selasky  * 2. Redistributions in binary form must reproduce the above copyright
12*1a01b4e5SHans Petter Selasky  *    notice, this list of conditions and the following disclaimer in the
13*1a01b4e5SHans Petter Selasky  *    documentation and/or other materials provided with the distribution.
14*1a01b4e5SHans Petter Selasky  *
15*1a01b4e5SHans Petter Selasky  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16*1a01b4e5SHans Petter Selasky  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17*1a01b4e5SHans Petter Selasky  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18*1a01b4e5SHans Petter Selasky  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19*1a01b4e5SHans Petter Selasky  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20*1a01b4e5SHans Petter Selasky  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21*1a01b4e5SHans Petter Selasky  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22*1a01b4e5SHans Petter Selasky  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23*1a01b4e5SHans Petter Selasky  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24*1a01b4e5SHans Petter Selasky  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25*1a01b4e5SHans Petter Selasky  */
26*1a01b4e5SHans Petter Selasky 
27*1a01b4e5SHans Petter Selasky #include <sys/cdefs.h>
28*1a01b4e5SHans Petter Selasky __FBSDID("$FreeBSD$");
29*1a01b4e5SHans Petter Selasky 
30*1a01b4e5SHans Petter Selasky #include <sys/types.h>
31*1a01b4e5SHans Petter Selasky #include <sys/systm.h>
32*1a01b4e5SHans Petter Selasky #include <sys/malloc.h>
33*1a01b4e5SHans Petter Selasky #include <sys/kernel.h>
34*1a01b4e5SHans Petter Selasky #include <sys/lock.h>
35*1a01b4e5SHans Petter Selasky #include <sys/mutex.h>
36*1a01b4e5SHans Petter Selasky #include <sys/proc.h>
37*1a01b4e5SHans Petter Selasky #include <sys/sched.h>
38*1a01b4e5SHans Petter Selasky #include <sys/smp.h>
39*1a01b4e5SHans Petter Selasky #include <sys/queue.h>
40*1a01b4e5SHans Petter Selasky #include <sys/taskqueue.h>
41*1a01b4e5SHans Petter Selasky 
42*1a01b4e5SHans Petter Selasky #include <ck_epoch.h>
43*1a01b4e5SHans Petter Selasky 
44*1a01b4e5SHans Petter Selasky #include <linux/rcupdate.h>
45*1a01b4e5SHans Petter Selasky #include <linux/srcu.h>
46*1a01b4e5SHans Petter Selasky #include <linux/slab.h>
47*1a01b4e5SHans Petter Selasky #include <linux/kernel.h>
48*1a01b4e5SHans Petter Selasky 
49*1a01b4e5SHans Petter Selasky struct callback_head {
50*1a01b4e5SHans Petter Selasky 	ck_epoch_entry_t epoch_entry;
51*1a01b4e5SHans Petter Selasky 	rcu_callback_t func;
52*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *epoch_record;
53*1a01b4e5SHans Petter Selasky 	struct task task;
54*1a01b4e5SHans Petter Selasky };
55*1a01b4e5SHans Petter Selasky 
56*1a01b4e5SHans Petter Selasky /*
57*1a01b4e5SHans Petter Selasky  * Verify that "struct rcu_head" is big enough to hold "struct
58*1a01b4e5SHans Petter Selasky  * callback_head". This has been done to avoid having to add special
59*1a01b4e5SHans Petter Selasky  * compile flags for including ck_epoch.h to all clients of the
60*1a01b4e5SHans Petter Selasky  * LinuxKPI.
61*1a01b4e5SHans Petter Selasky  */
62*1a01b4e5SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) >= sizeof(struct callback_head));
63*1a01b4e5SHans Petter Selasky 
64*1a01b4e5SHans Petter Selasky static ck_epoch_t linux_epoch;
65*1a01b4e5SHans Petter Selasky static	MALLOC_DEFINE(M_LRCU, "lrcu", "Linux RCU");
66*1a01b4e5SHans Petter Selasky static	DPCPU_DEFINE(ck_epoch_record_t *, epoch_record);
67*1a01b4e5SHans Petter Selasky 
68*1a01b4e5SHans Petter Selasky static void
69*1a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused)
70*1a01b4e5SHans Petter Selasky {
71*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t **pcpu_record;
72*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
73*1a01b4e5SHans Petter Selasky 	int i;
74*1a01b4e5SHans Petter Selasky 
75*1a01b4e5SHans Petter Selasky 	ck_epoch_init(&linux_epoch);
76*1a01b4e5SHans Petter Selasky 
77*1a01b4e5SHans Petter Selasky 	CPU_FOREACH(i) {
78*1a01b4e5SHans Petter Selasky 		record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
79*1a01b4e5SHans Petter Selasky 		ck_epoch_register(&linux_epoch, record);
80*1a01b4e5SHans Petter Selasky 		pcpu_record = DPCPU_ID_PTR(i, epoch_record);
81*1a01b4e5SHans Petter Selasky 		*pcpu_record = record;
82*1a01b4e5SHans Petter Selasky 	}
83*1a01b4e5SHans Petter Selasky 
84*1a01b4e5SHans Petter Selasky 	/*
85*1a01b4e5SHans Petter Selasky 	 * Populate the epoch with 5 * ncpus # of records
86*1a01b4e5SHans Petter Selasky 	 */
87*1a01b4e5SHans Petter Selasky 	for (i = 0; i < 5 * mp_ncpus; i++) {
88*1a01b4e5SHans Petter Selasky 		record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
89*1a01b4e5SHans Petter Selasky 		ck_epoch_register(&linux_epoch, record);
90*1a01b4e5SHans Petter Selasky 		ck_epoch_unregister(record);
91*1a01b4e5SHans Petter Selasky 	}
92*1a01b4e5SHans Petter Selasky }
93*1a01b4e5SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_init, NULL);
94*1a01b4e5SHans Petter Selasky 
95*1a01b4e5SHans Petter Selasky static void
96*1a01b4e5SHans Petter Selasky linux_rcu_runtime_uninit(void *arg __unused)
97*1a01b4e5SHans Petter Selasky {
98*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t **pcpu_record;
99*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
100*1a01b4e5SHans Petter Selasky 	int i;
101*1a01b4e5SHans Petter Selasky 
102*1a01b4e5SHans Petter Selasky 	while ((record = ck_epoch_recycle(&linux_epoch)) != NULL)
103*1a01b4e5SHans Petter Selasky 		free(record, M_LRCU);
104*1a01b4e5SHans Petter Selasky 
105*1a01b4e5SHans Petter Selasky 	CPU_FOREACH(i) {
106*1a01b4e5SHans Petter Selasky 		pcpu_record = DPCPU_ID_PTR(i, epoch_record);
107*1a01b4e5SHans Petter Selasky 		record = *pcpu_record;
108*1a01b4e5SHans Petter Selasky 		*pcpu_record = NULL;
109*1a01b4e5SHans Petter Selasky 		free(record, M_LRCU);
110*1a01b4e5SHans Petter Selasky 	}
111*1a01b4e5SHans Petter Selasky }
112*1a01b4e5SHans Petter Selasky SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);
113*1a01b4e5SHans Petter Selasky 
114*1a01b4e5SHans Petter Selasky static ck_epoch_record_t *
115*1a01b4e5SHans Petter Selasky linux_rcu_get_record(int canblock)
116*1a01b4e5SHans Petter Selasky {
117*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
118*1a01b4e5SHans Petter Selasky 
119*1a01b4e5SHans Petter Selasky 	if (__predict_true((record = ck_epoch_recycle(&linux_epoch)) != NULL))
120*1a01b4e5SHans Petter Selasky 		return (record);
121*1a01b4e5SHans Petter Selasky 	if ((record = malloc(sizeof(*record), M_LRCU, M_NOWAIT | M_ZERO)) != NULL) {
122*1a01b4e5SHans Petter Selasky 		ck_epoch_register(&linux_epoch, record);
123*1a01b4e5SHans Petter Selasky 		return (record);
124*1a01b4e5SHans Petter Selasky 	} else if (!canblock)
125*1a01b4e5SHans Petter Selasky 		return (NULL);
126*1a01b4e5SHans Petter Selasky 
127*1a01b4e5SHans Petter Selasky 	record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
128*1a01b4e5SHans Petter Selasky 	ck_epoch_register(&linux_epoch, record);
129*1a01b4e5SHans Petter Selasky 	return (record);
130*1a01b4e5SHans Petter Selasky }
131*1a01b4e5SHans Petter Selasky 
132*1a01b4e5SHans Petter Selasky static void
133*1a01b4e5SHans Petter Selasky linux_rcu_destroy_object(ck_epoch_entry_t *e)
134*1a01b4e5SHans Petter Selasky {
135*1a01b4e5SHans Petter Selasky 	struct callback_head *rcu;
136*1a01b4e5SHans Petter Selasky 	uintptr_t offset;
137*1a01b4e5SHans Petter Selasky 
138*1a01b4e5SHans Petter Selasky 	rcu = container_of(e, struct callback_head, epoch_entry);
139*1a01b4e5SHans Petter Selasky 
140*1a01b4e5SHans Petter Selasky 	offset = (uintptr_t)rcu->func;
141*1a01b4e5SHans Petter Selasky 
142*1a01b4e5SHans Petter Selasky 	MPASS(rcu->task.ta_pending == 0);
143*1a01b4e5SHans Petter Selasky 
144*1a01b4e5SHans Petter Selasky 	if (offset < LINUX_KFREE_RCU_OFFSET_MAX)
145*1a01b4e5SHans Petter Selasky 		kfree((char *)rcu - offset);
146*1a01b4e5SHans Petter Selasky 	else
147*1a01b4e5SHans Petter Selasky 		rcu->func((struct rcu_head *)rcu);
148*1a01b4e5SHans Petter Selasky }
149*1a01b4e5SHans Petter Selasky 
150*1a01b4e5SHans Petter Selasky static void
151*1a01b4e5SHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused)
152*1a01b4e5SHans Petter Selasky {
153*1a01b4e5SHans Petter Selasky 	struct callback_head *rcu = context;
154*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record = rcu->epoch_record;
155*1a01b4e5SHans Petter Selasky 
156*1a01b4e5SHans Petter Selasky 	ck_epoch_barrier(record);
157*1a01b4e5SHans Petter Selasky 	ck_epoch_unregister(record);
158*1a01b4e5SHans Petter Selasky }
159*1a01b4e5SHans Petter Selasky 
160*1a01b4e5SHans Petter Selasky void
161*1a01b4e5SHans Petter Selasky linux_rcu_read_lock(void)
162*1a01b4e5SHans Petter Selasky {
163*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
164*1a01b4e5SHans Petter Selasky 
165*1a01b4e5SHans Petter Selasky 	sched_pin();
166*1a01b4e5SHans Petter Selasky 	record = DPCPU_GET(epoch_record);
167*1a01b4e5SHans Petter Selasky 	MPASS(record != NULL);
168*1a01b4e5SHans Petter Selasky 
169*1a01b4e5SHans Petter Selasky 	ck_epoch_begin(record, NULL);
170*1a01b4e5SHans Petter Selasky }
171*1a01b4e5SHans Petter Selasky 
172*1a01b4e5SHans Petter Selasky void
173*1a01b4e5SHans Petter Selasky linux_rcu_read_unlock(void)
174*1a01b4e5SHans Petter Selasky {
175*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
176*1a01b4e5SHans Petter Selasky 
177*1a01b4e5SHans Petter Selasky 	record = DPCPU_GET(epoch_record);
178*1a01b4e5SHans Petter Selasky 	ck_epoch_end(record, NULL);
179*1a01b4e5SHans Petter Selasky 	sched_unpin();
180*1a01b4e5SHans Petter Selasky }
181*1a01b4e5SHans Petter Selasky 
182*1a01b4e5SHans Petter Selasky void
183*1a01b4e5SHans Petter Selasky linux_synchronize_rcu(void)
184*1a01b4e5SHans Petter Selasky {
185*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
186*1a01b4e5SHans Petter Selasky 
187*1a01b4e5SHans Petter Selasky 	sched_pin();
188*1a01b4e5SHans Petter Selasky 	record = DPCPU_GET(epoch_record);
189*1a01b4e5SHans Petter Selasky 	MPASS(record != NULL);
190*1a01b4e5SHans Petter Selasky 	ck_epoch_synchronize(record);
191*1a01b4e5SHans Petter Selasky 	sched_unpin();
192*1a01b4e5SHans Petter Selasky }
193*1a01b4e5SHans Petter Selasky 
194*1a01b4e5SHans Petter Selasky void
195*1a01b4e5SHans Petter Selasky linux_rcu_barrier(void)
196*1a01b4e5SHans Petter Selasky {
197*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
198*1a01b4e5SHans Petter Selasky 
199*1a01b4e5SHans Petter Selasky 	record = linux_rcu_get_record(0);
200*1a01b4e5SHans Petter Selasky 	ck_epoch_barrier(record);
201*1a01b4e5SHans Petter Selasky 	ck_epoch_unregister(record);
202*1a01b4e5SHans Petter Selasky }
203*1a01b4e5SHans Petter Selasky 
204*1a01b4e5SHans Petter Selasky void
205*1a01b4e5SHans Petter Selasky linux_call_rcu(struct rcu_head *context, rcu_callback_t func)
206*1a01b4e5SHans Petter Selasky {
207*1a01b4e5SHans Petter Selasky 	struct callback_head *ptr = (struct callback_head *)context;
208*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
209*1a01b4e5SHans Petter Selasky 
210*1a01b4e5SHans Petter Selasky 	record = linux_rcu_get_record(0);
211*1a01b4e5SHans Petter Selasky 
212*1a01b4e5SHans Petter Selasky 	sched_pin();
213*1a01b4e5SHans Petter Selasky 	MPASS(record != NULL);
214*1a01b4e5SHans Petter Selasky 	ptr->func = func;
215*1a01b4e5SHans Petter Selasky 	ptr->epoch_record = record;
216*1a01b4e5SHans Petter Selasky 	ck_epoch_call(record, &ptr->epoch_entry, linux_rcu_destroy_object);
217*1a01b4e5SHans Petter Selasky 	TASK_INIT(&ptr->task, 0, linux_rcu_cleaner_func, ptr);
218*1a01b4e5SHans Petter Selasky 	taskqueue_enqueue(taskqueue_fast, &ptr->task);
219*1a01b4e5SHans Petter Selasky 	sched_unpin();
220*1a01b4e5SHans Petter Selasky }
221*1a01b4e5SHans Petter Selasky 
222*1a01b4e5SHans Petter Selasky int
223*1a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu)
224*1a01b4e5SHans Petter Selasky {
225*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
226*1a01b4e5SHans Petter Selasky 
227*1a01b4e5SHans Petter Selasky 	record = linux_rcu_get_record(0);
228*1a01b4e5SHans Petter Selasky 	srcu->ss_epoch_record = record;
229*1a01b4e5SHans Petter Selasky 	return (0);
230*1a01b4e5SHans Petter Selasky }
231*1a01b4e5SHans Petter Selasky 
232*1a01b4e5SHans Petter Selasky void
233*1a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu)
234*1a01b4e5SHans Petter Selasky {
235*1a01b4e5SHans Petter Selasky 	ck_epoch_record_t *record;
236*1a01b4e5SHans Petter Selasky 
237*1a01b4e5SHans Petter Selasky 	record = srcu->ss_epoch_record;
238*1a01b4e5SHans Petter Selasky 	srcu->ss_epoch_record = NULL;
239*1a01b4e5SHans Petter Selasky 	ck_epoch_unregister(record);
240*1a01b4e5SHans Petter Selasky }
241*1a01b4e5SHans Petter Selasky 
242*1a01b4e5SHans Petter Selasky int
243*1a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu)
244*1a01b4e5SHans Petter Selasky {
245*1a01b4e5SHans Petter Selasky 	ck_epoch_begin(srcu->ss_epoch_record, NULL);
246*1a01b4e5SHans Petter Selasky 	return (0);
247*1a01b4e5SHans Petter Selasky }
248*1a01b4e5SHans Petter Selasky 
249*1a01b4e5SHans Petter Selasky void
250*1a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
251*1a01b4e5SHans Petter Selasky {
252*1a01b4e5SHans Petter Selasky 	ck_epoch_end(srcu->ss_epoch_record, NULL);
253*1a01b4e5SHans Petter Selasky }
254*1a01b4e5SHans Petter Selasky 
255*1a01b4e5SHans Petter Selasky void
256*1a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu)
257*1a01b4e5SHans Petter Selasky {
258*1a01b4e5SHans Petter Selasky 	ck_epoch_synchronize(srcu->ss_epoch_record);
259*1a01b4e5SHans Petter Selasky }
260