1*1a01b4e5SHans Petter Selasky /*- 2*1a01b4e5SHans Petter Selasky * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org) 3*1a01b4e5SHans Petter Selasky * All rights reserved. 4*1a01b4e5SHans Petter Selasky * 5*1a01b4e5SHans Petter Selasky * Redistribution and use in source and binary forms, with or without 6*1a01b4e5SHans Petter Selasky * modification, are permitted provided that the following conditions 7*1a01b4e5SHans Petter Selasky * are met: 8*1a01b4e5SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright 9*1a01b4e5SHans Petter Selasky * notice unmodified, this list of conditions, and the following 10*1a01b4e5SHans Petter Selasky * disclaimer. 11*1a01b4e5SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright 12*1a01b4e5SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the 13*1a01b4e5SHans Petter Selasky * documentation and/or other materials provided with the distribution. 14*1a01b4e5SHans Petter Selasky * 15*1a01b4e5SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16*1a01b4e5SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17*1a01b4e5SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18*1a01b4e5SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19*1a01b4e5SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20*1a01b4e5SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21*1a01b4e5SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22*1a01b4e5SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23*1a01b4e5SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24*1a01b4e5SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25*1a01b4e5SHans Petter Selasky */ 26*1a01b4e5SHans Petter Selasky 27*1a01b4e5SHans Petter Selasky #include <sys/cdefs.h> 28*1a01b4e5SHans Petter Selasky __FBSDID("$FreeBSD$"); 29*1a01b4e5SHans Petter Selasky 30*1a01b4e5SHans Petter Selasky #include <sys/types.h> 31*1a01b4e5SHans Petter Selasky #include <sys/systm.h> 32*1a01b4e5SHans Petter Selasky #include <sys/malloc.h> 33*1a01b4e5SHans Petter Selasky #include <sys/kernel.h> 34*1a01b4e5SHans Petter Selasky #include <sys/lock.h> 35*1a01b4e5SHans Petter Selasky #include <sys/mutex.h> 36*1a01b4e5SHans Petter Selasky #include <sys/proc.h> 37*1a01b4e5SHans Petter Selasky #include <sys/sched.h> 38*1a01b4e5SHans Petter Selasky #include <sys/smp.h> 39*1a01b4e5SHans Petter Selasky #include <sys/queue.h> 40*1a01b4e5SHans Petter Selasky #include <sys/taskqueue.h> 41*1a01b4e5SHans Petter Selasky 42*1a01b4e5SHans Petter Selasky #include <ck_epoch.h> 43*1a01b4e5SHans Petter Selasky 44*1a01b4e5SHans Petter Selasky #include <linux/rcupdate.h> 45*1a01b4e5SHans Petter Selasky #include <linux/srcu.h> 46*1a01b4e5SHans Petter Selasky #include <linux/slab.h> 47*1a01b4e5SHans Petter Selasky #include <linux/kernel.h> 48*1a01b4e5SHans Petter Selasky 49*1a01b4e5SHans Petter Selasky struct callback_head { 50*1a01b4e5SHans Petter Selasky ck_epoch_entry_t epoch_entry; 51*1a01b4e5SHans Petter Selasky rcu_callback_t func; 52*1a01b4e5SHans Petter Selasky ck_epoch_record_t *epoch_record; 53*1a01b4e5SHans Petter Selasky struct task task; 54*1a01b4e5SHans Petter Selasky }; 55*1a01b4e5SHans Petter Selasky 56*1a01b4e5SHans Petter Selasky /* 57*1a01b4e5SHans Petter Selasky * Verify that "struct rcu_head" is big enough to hold "struct 58*1a01b4e5SHans Petter Selasky * callback_head". This has been done to avoid having to add special 59*1a01b4e5SHans Petter Selasky * compile flags for including ck_epoch.h to all clients of the 60*1a01b4e5SHans Petter Selasky * LinuxKPI. 61*1a01b4e5SHans Petter Selasky */ 62*1a01b4e5SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) >= sizeof(struct callback_head)); 63*1a01b4e5SHans Petter Selasky 64*1a01b4e5SHans Petter Selasky static ck_epoch_t linux_epoch; 65*1a01b4e5SHans Petter Selasky static MALLOC_DEFINE(M_LRCU, "lrcu", "Linux RCU"); 66*1a01b4e5SHans Petter Selasky static DPCPU_DEFINE(ck_epoch_record_t *, epoch_record); 67*1a01b4e5SHans Petter Selasky 68*1a01b4e5SHans Petter Selasky static void 69*1a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused) 70*1a01b4e5SHans Petter Selasky { 71*1a01b4e5SHans Petter Selasky ck_epoch_record_t **pcpu_record; 72*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 73*1a01b4e5SHans Petter Selasky int i; 74*1a01b4e5SHans Petter Selasky 75*1a01b4e5SHans Petter Selasky ck_epoch_init(&linux_epoch); 76*1a01b4e5SHans Petter Selasky 77*1a01b4e5SHans Petter Selasky CPU_FOREACH(i) { 78*1a01b4e5SHans Petter Selasky record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 79*1a01b4e5SHans Petter Selasky ck_epoch_register(&linux_epoch, record); 80*1a01b4e5SHans Petter Selasky pcpu_record = DPCPU_ID_PTR(i, epoch_record); 81*1a01b4e5SHans Petter Selasky *pcpu_record = record; 82*1a01b4e5SHans Petter Selasky } 83*1a01b4e5SHans Petter Selasky 84*1a01b4e5SHans Petter Selasky /* 85*1a01b4e5SHans Petter Selasky * Populate the epoch with 5 * ncpus # of records 86*1a01b4e5SHans Petter Selasky */ 87*1a01b4e5SHans Petter Selasky for (i = 0; i < 5 * mp_ncpus; i++) { 88*1a01b4e5SHans Petter Selasky record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 89*1a01b4e5SHans Petter Selasky ck_epoch_register(&linux_epoch, record); 90*1a01b4e5SHans Petter Selasky ck_epoch_unregister(record); 91*1a01b4e5SHans Petter Selasky } 92*1a01b4e5SHans Petter Selasky } 93*1a01b4e5SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_init, NULL); 94*1a01b4e5SHans Petter Selasky 95*1a01b4e5SHans Petter Selasky static void 96*1a01b4e5SHans Petter Selasky linux_rcu_runtime_uninit(void *arg __unused) 97*1a01b4e5SHans Petter Selasky { 98*1a01b4e5SHans Petter Selasky ck_epoch_record_t **pcpu_record; 99*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 100*1a01b4e5SHans Petter Selasky int i; 101*1a01b4e5SHans Petter Selasky 102*1a01b4e5SHans Petter Selasky while ((record = ck_epoch_recycle(&linux_epoch)) != NULL) 103*1a01b4e5SHans Petter Selasky free(record, M_LRCU); 104*1a01b4e5SHans Petter Selasky 105*1a01b4e5SHans Petter Selasky CPU_FOREACH(i) { 106*1a01b4e5SHans Petter Selasky pcpu_record = DPCPU_ID_PTR(i, epoch_record); 107*1a01b4e5SHans Petter Selasky record = *pcpu_record; 108*1a01b4e5SHans Petter Selasky *pcpu_record = NULL; 109*1a01b4e5SHans Petter Selasky free(record, M_LRCU); 110*1a01b4e5SHans Petter Selasky } 111*1a01b4e5SHans Petter Selasky } 112*1a01b4e5SHans Petter Selasky SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); 113*1a01b4e5SHans Petter Selasky 114*1a01b4e5SHans Petter Selasky static ck_epoch_record_t * 115*1a01b4e5SHans Petter Selasky linux_rcu_get_record(int canblock) 116*1a01b4e5SHans Petter Selasky { 117*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 118*1a01b4e5SHans Petter Selasky 119*1a01b4e5SHans Petter Selasky if (__predict_true((record = ck_epoch_recycle(&linux_epoch)) != NULL)) 120*1a01b4e5SHans Petter Selasky return (record); 121*1a01b4e5SHans Petter Selasky if ((record = malloc(sizeof(*record), M_LRCU, M_NOWAIT | M_ZERO)) != NULL) { 122*1a01b4e5SHans Petter Selasky ck_epoch_register(&linux_epoch, record); 123*1a01b4e5SHans Petter Selasky return (record); 124*1a01b4e5SHans Petter Selasky } else if (!canblock) 125*1a01b4e5SHans Petter Selasky return (NULL); 126*1a01b4e5SHans Petter Selasky 127*1a01b4e5SHans Petter Selasky record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 128*1a01b4e5SHans Petter Selasky ck_epoch_register(&linux_epoch, record); 129*1a01b4e5SHans Petter Selasky return (record); 130*1a01b4e5SHans Petter Selasky } 131*1a01b4e5SHans Petter Selasky 132*1a01b4e5SHans Petter Selasky static void 133*1a01b4e5SHans Petter Selasky linux_rcu_destroy_object(ck_epoch_entry_t *e) 134*1a01b4e5SHans Petter Selasky { 135*1a01b4e5SHans Petter Selasky struct callback_head *rcu; 136*1a01b4e5SHans Petter Selasky uintptr_t offset; 137*1a01b4e5SHans Petter Selasky 138*1a01b4e5SHans Petter Selasky rcu = container_of(e, struct callback_head, epoch_entry); 139*1a01b4e5SHans Petter Selasky 140*1a01b4e5SHans Petter Selasky offset = (uintptr_t)rcu->func; 141*1a01b4e5SHans Petter Selasky 142*1a01b4e5SHans Petter Selasky MPASS(rcu->task.ta_pending == 0); 143*1a01b4e5SHans Petter Selasky 144*1a01b4e5SHans Petter Selasky if (offset < LINUX_KFREE_RCU_OFFSET_MAX) 145*1a01b4e5SHans Petter Selasky kfree((char *)rcu - offset); 146*1a01b4e5SHans Petter Selasky else 147*1a01b4e5SHans Petter Selasky rcu->func((struct rcu_head *)rcu); 148*1a01b4e5SHans Petter Selasky } 149*1a01b4e5SHans Petter Selasky 150*1a01b4e5SHans Petter Selasky static void 151*1a01b4e5SHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused) 152*1a01b4e5SHans Petter Selasky { 153*1a01b4e5SHans Petter Selasky struct callback_head *rcu = context; 154*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record = rcu->epoch_record; 155*1a01b4e5SHans Petter Selasky 156*1a01b4e5SHans Petter Selasky ck_epoch_barrier(record); 157*1a01b4e5SHans Petter Selasky ck_epoch_unregister(record); 158*1a01b4e5SHans Petter Selasky } 159*1a01b4e5SHans Petter Selasky 160*1a01b4e5SHans Petter Selasky void 161*1a01b4e5SHans Petter Selasky linux_rcu_read_lock(void) 162*1a01b4e5SHans Petter Selasky { 163*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 164*1a01b4e5SHans Petter Selasky 165*1a01b4e5SHans Petter Selasky sched_pin(); 166*1a01b4e5SHans Petter Selasky record = DPCPU_GET(epoch_record); 167*1a01b4e5SHans Petter Selasky MPASS(record != NULL); 168*1a01b4e5SHans Petter Selasky 169*1a01b4e5SHans Petter Selasky ck_epoch_begin(record, NULL); 170*1a01b4e5SHans Petter Selasky } 171*1a01b4e5SHans Petter Selasky 172*1a01b4e5SHans Petter Selasky void 173*1a01b4e5SHans Petter Selasky linux_rcu_read_unlock(void) 174*1a01b4e5SHans Petter Selasky { 175*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 176*1a01b4e5SHans Petter Selasky 177*1a01b4e5SHans Petter Selasky record = DPCPU_GET(epoch_record); 178*1a01b4e5SHans Petter Selasky ck_epoch_end(record, NULL); 179*1a01b4e5SHans Petter Selasky sched_unpin(); 180*1a01b4e5SHans Petter Selasky } 181*1a01b4e5SHans Petter Selasky 182*1a01b4e5SHans Petter Selasky void 183*1a01b4e5SHans Petter Selasky linux_synchronize_rcu(void) 184*1a01b4e5SHans Petter Selasky { 185*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 186*1a01b4e5SHans Petter Selasky 187*1a01b4e5SHans Petter Selasky sched_pin(); 188*1a01b4e5SHans Petter Selasky record = DPCPU_GET(epoch_record); 189*1a01b4e5SHans Petter Selasky MPASS(record != NULL); 190*1a01b4e5SHans Petter Selasky ck_epoch_synchronize(record); 191*1a01b4e5SHans Petter Selasky sched_unpin(); 192*1a01b4e5SHans Petter Selasky } 193*1a01b4e5SHans Petter Selasky 194*1a01b4e5SHans Petter Selasky void 195*1a01b4e5SHans Petter Selasky linux_rcu_barrier(void) 196*1a01b4e5SHans Petter Selasky { 197*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 198*1a01b4e5SHans Petter Selasky 199*1a01b4e5SHans Petter Selasky record = linux_rcu_get_record(0); 200*1a01b4e5SHans Petter Selasky ck_epoch_barrier(record); 201*1a01b4e5SHans Petter Selasky ck_epoch_unregister(record); 202*1a01b4e5SHans Petter Selasky } 203*1a01b4e5SHans Petter Selasky 204*1a01b4e5SHans Petter Selasky void 205*1a01b4e5SHans Petter Selasky linux_call_rcu(struct rcu_head *context, rcu_callback_t func) 206*1a01b4e5SHans Petter Selasky { 207*1a01b4e5SHans Petter Selasky struct callback_head *ptr = (struct callback_head *)context; 208*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 209*1a01b4e5SHans Petter Selasky 210*1a01b4e5SHans Petter Selasky record = linux_rcu_get_record(0); 211*1a01b4e5SHans Petter Selasky 212*1a01b4e5SHans Petter Selasky sched_pin(); 213*1a01b4e5SHans Petter Selasky MPASS(record != NULL); 214*1a01b4e5SHans Petter Selasky ptr->func = func; 215*1a01b4e5SHans Petter Selasky ptr->epoch_record = record; 216*1a01b4e5SHans Petter Selasky ck_epoch_call(record, &ptr->epoch_entry, linux_rcu_destroy_object); 217*1a01b4e5SHans Petter Selasky TASK_INIT(&ptr->task, 0, linux_rcu_cleaner_func, ptr); 218*1a01b4e5SHans Petter Selasky taskqueue_enqueue(taskqueue_fast, &ptr->task); 219*1a01b4e5SHans Petter Selasky sched_unpin(); 220*1a01b4e5SHans Petter Selasky } 221*1a01b4e5SHans Petter Selasky 222*1a01b4e5SHans Petter Selasky int 223*1a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu) 224*1a01b4e5SHans Petter Selasky { 225*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 226*1a01b4e5SHans Petter Selasky 227*1a01b4e5SHans Petter Selasky record = linux_rcu_get_record(0); 228*1a01b4e5SHans Petter Selasky srcu->ss_epoch_record = record; 229*1a01b4e5SHans Petter Selasky return (0); 230*1a01b4e5SHans Petter Selasky } 231*1a01b4e5SHans Petter Selasky 232*1a01b4e5SHans Petter Selasky void 233*1a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu) 234*1a01b4e5SHans Petter Selasky { 235*1a01b4e5SHans Petter Selasky ck_epoch_record_t *record; 236*1a01b4e5SHans Petter Selasky 237*1a01b4e5SHans Petter Selasky record = srcu->ss_epoch_record; 238*1a01b4e5SHans Petter Selasky srcu->ss_epoch_record = NULL; 239*1a01b4e5SHans Petter Selasky ck_epoch_unregister(record); 240*1a01b4e5SHans Petter Selasky } 241*1a01b4e5SHans Petter Selasky 242*1a01b4e5SHans Petter Selasky int 243*1a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu) 244*1a01b4e5SHans Petter Selasky { 245*1a01b4e5SHans Petter Selasky ck_epoch_begin(srcu->ss_epoch_record, NULL); 246*1a01b4e5SHans Petter Selasky return (0); 247*1a01b4e5SHans Petter Selasky } 248*1a01b4e5SHans Petter Selasky 249*1a01b4e5SHans Petter Selasky void 250*1a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused) 251*1a01b4e5SHans Petter Selasky { 252*1a01b4e5SHans Petter Selasky ck_epoch_end(srcu->ss_epoch_record, NULL); 253*1a01b4e5SHans Petter Selasky } 254*1a01b4e5SHans Petter Selasky 255*1a01b4e5SHans Petter Selasky void 256*1a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu) 257*1a01b4e5SHans Petter Selasky { 258*1a01b4e5SHans Petter Selasky ck_epoch_synchronize(srcu->ss_epoch_record); 259*1a01b4e5SHans Petter Selasky } 260