1*77a6b00eSFrançois Tigeot /* 2*77a6b00eSFrançois Tigeot * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org> 3*77a6b00eSFrançois Tigeot * All rights reserved. 4*77a6b00eSFrançois Tigeot * 5*77a6b00eSFrançois Tigeot * Redistribution and use in source and binary forms, with or without 6*77a6b00eSFrançois Tigeot * modification, are permitted provided that the following conditions 7*77a6b00eSFrançois Tigeot * are met: 8*77a6b00eSFrançois Tigeot * 1. Redistributions of source code must retain the above copyright 9*77a6b00eSFrançois Tigeot * notice unmodified, this list of conditions, and the following 10*77a6b00eSFrançois Tigeot * disclaimer. 11*77a6b00eSFrançois Tigeot * 2. Redistributions in binary form must reproduce the above copyright 12*77a6b00eSFrançois Tigeot * notice, this list of conditions and the following disclaimer in the 13*77a6b00eSFrançois Tigeot * documentation and/or other materials provided with the distribution. 14*77a6b00eSFrançois Tigeot * 15*77a6b00eSFrançois Tigeot * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16*77a6b00eSFrançois Tigeot * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17*77a6b00eSFrançois Tigeot * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18*77a6b00eSFrançois Tigeot * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19*77a6b00eSFrançois Tigeot * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20*77a6b00eSFrançois Tigeot * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21*77a6b00eSFrançois Tigeot * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22*77a6b00eSFrançois Tigeot * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23*77a6b00eSFrançois Tigeot * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24*77a6b00eSFrançois Tigeot * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25*77a6b00eSFrançois Tigeot */ 26*77a6b00eSFrançois Tigeot 27*77a6b00eSFrançois Tigeot #include <linux/interrupt.h> 28*77a6b00eSFrançois Tigeot #include <linux/slab.h> 29*77a6b00eSFrançois Tigeot 30*77a6b00eSFrançois Tigeot #include <sys/kthread.h> 31*77a6b00eSFrançois Tigeot 32*77a6b00eSFrançois Tigeot /* 33*77a6b00eSFrançois Tigeot * Linux tasklet constraints: 34*77a6b00eSFrançois Tigeot * - tasklets that have the same type cannot be run on multiple processors at 35*77a6b00eSFrançois Tigeot * the same time 36*77a6b00eSFrançois Tigeot * - tasklets always run on the processor from which they were originally 37*77a6b00eSFrançois Tigeot * submitted 38*77a6b00eSFrançois Tigeot * - when a tasklet is scheduled, its state is set to TASKLET_STATE_SCHED, 39*77a6b00eSFrançois Tigeot * and the tasklet added to a queue 40*77a6b00eSFrançois Tigeot * - during the execution of its function, the tasklet state is set to 41*77a6b00eSFrançois Tigeot * TASKLET_STATE_RUN and the TASKLET_STATE_SCHED state is removed 42*77a6b00eSFrançois Tigeot */ 43*77a6b00eSFrançois Tigeot 44*77a6b00eSFrançois Tigeot struct tasklet_entry { 45*77a6b00eSFrançois Tigeot struct tasklet_struct *ts; 46*77a6b00eSFrançois Tigeot SLIST_ENTRY(tasklet_entry) tasklet_entries; 47*77a6b00eSFrançois Tigeot }; 48*77a6b00eSFrançois Tigeot 49*77a6b00eSFrançois Tigeot static struct lock tasklet_lock = LOCK_INITIALIZER("dltll", 0, LK_CANRECURSE); 50*77a6b00eSFrançois Tigeot 51*77a6b00eSFrançois Tigeot static struct thread *tasklet_td = NULL; 52*77a6b00eSFrançois Tigeot SLIST_HEAD(tasklet_list_head, tasklet_entry) tlist = SLIST_HEAD_INITIALIZER(tlist); 53*77a6b00eSFrançois Tigeot SLIST_HEAD(tasklet_hi_list_head, tasklet_entry) tlist_hi = SLIST_HEAD_INITIALIZER(tlist_hi); 54*77a6b00eSFrançois Tigeot 55*77a6b00eSFrançois Tigeot static int tasklet_pending = 0; 56*77a6b00eSFrançois Tigeot 57*77a6b00eSFrançois Tigeot #define PROCESS_TASKLET_LIST(which_list) do { \ 58*77a6b00eSFrançois Tigeot SLIST_FOREACH_MUTABLE(te, &which_list, tasklet_entries, tmp_te) { \ 59*77a6b00eSFrançois Tigeot struct tasklet_struct *t = te->ts; \ 60*77a6b00eSFrançois Tigeot \ 61*77a6b00eSFrançois Tigeot /* \ 62*77a6b00eSFrançois Tigeot This tasklet is dying, remove it from the list. \ 63*77a6b00eSFrançois Tigeot We allow to it to run one last time if it has \ 64*77a6b00eSFrançois Tigeot already been scheduled. \ 65*77a6b00eSFrançois Tigeot */ \ 66*77a6b00eSFrançois Tigeot if (test_bit(TASKLET_IS_DYING, &t->state)) { \ 67*77a6b00eSFrançois Tigeot kprintf("tasklet: killing %p\n", t); \ 68*77a6b00eSFrançois Tigeot SLIST_REMOVE(&which_list, te, tasklet_entry, tasklet_entries); \ 69*77a6b00eSFrançois Tigeot kfree(te); \ 70*77a6b00eSFrançois Tigeot } \ 71*77a6b00eSFrançois Tigeot \ 72*77a6b00eSFrançois Tigeot /* This tasklet is not scheduled, try the next one */ \ 73*77a6b00eSFrançois Tigeot if (!test_bit(TASKLET_STATE_SCHED, &t->state)) \ 74*77a6b00eSFrançois Tigeot continue; \ 75*77a6b00eSFrançois Tigeot \ 76*77a6b00eSFrançois Tigeot clear_bit(TASKLET_STATE_SCHED, &t->state); \ 77*77a6b00eSFrançois Tigeot set_bit(TASKLET_STATE_RUN, &t->state); \ 78*77a6b00eSFrançois Tigeot \ 79*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_RELEASE); \ 80*77a6b00eSFrançois Tigeot if (t->func) \ 81*77a6b00eSFrançois Tigeot t->func(t->data); \ 82*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_EXCLUSIVE); \ 83*77a6b00eSFrançois Tigeot \ 84*77a6b00eSFrançois Tigeot clear_bit(TASKLET_STATE_RUN, &t->state); \ 85*77a6b00eSFrançois Tigeot } \ 86*77a6b00eSFrançois Tigeot } while (0) 87*77a6b00eSFrançois Tigeot 88*77a6b00eSFrançois Tigeot /* XXX runners should be CPU-specific */ 89*77a6b00eSFrançois Tigeot static void 90*77a6b00eSFrançois Tigeot tasklet_runner(void *arg) 91*77a6b00eSFrançois Tigeot { 92*77a6b00eSFrançois Tigeot struct tasklet_entry *te, *tmp_te; 93*77a6b00eSFrançois Tigeot 94*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_EXCLUSIVE); 95*77a6b00eSFrançois Tigeot while (1) { 96*77a6b00eSFrançois Tigeot /* 97*77a6b00eSFrançois Tigeot Only sleep if we haven't been raced by a _schedule() 98*77a6b00eSFrançois Tigeot call during an unlock window 99*77a6b00eSFrançois Tigeot */ 100*77a6b00eSFrançois Tigeot if (tasklet_pending == 0) { 101*77a6b00eSFrançois Tigeot lksleep(&tasklet_runner, &tasklet_lock, 0, "tkidle", 0); 102*77a6b00eSFrançois Tigeot } 103*77a6b00eSFrançois Tigeot tasklet_pending = 0; 104*77a6b00eSFrançois Tigeot 105*77a6b00eSFrançois Tigeot /* Process hi tasklets first */ 106*77a6b00eSFrançois Tigeot PROCESS_TASKLET_LIST(tlist_hi); 107*77a6b00eSFrançois Tigeot PROCESS_TASKLET_LIST(tlist); 108*77a6b00eSFrançois Tigeot } 109*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_RELEASE); 110*77a6b00eSFrançois Tigeot } 111*77a6b00eSFrançois Tigeot 112*77a6b00eSFrançois Tigeot void 113*77a6b00eSFrançois Tigeot tasklet_init(struct tasklet_struct *t, 114*77a6b00eSFrançois Tigeot void (*func)(unsigned long), unsigned long data) 115*77a6b00eSFrançois Tigeot { 116*77a6b00eSFrançois Tigeot t->state = 0; 117*77a6b00eSFrançois Tigeot t->func = func; 118*77a6b00eSFrançois Tigeot t->data = data; 119*77a6b00eSFrançois Tigeot } 120*77a6b00eSFrançois Tigeot 121*77a6b00eSFrançois Tigeot void 122*77a6b00eSFrançois Tigeot tasklet_schedule(struct tasklet_struct *t) 123*77a6b00eSFrançois Tigeot { 124*77a6b00eSFrançois Tigeot struct tasklet_entry *te; 125*77a6b00eSFrançois Tigeot 126*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_EXCLUSIVE); 127*77a6b00eSFrançois Tigeot set_bit(TASKLET_STATE_SCHED, &t->state); 128*77a6b00eSFrançois Tigeot 129*77a6b00eSFrançois Tigeot SLIST_FOREACH(te, &tlist, tasklet_entries) { 130*77a6b00eSFrançois Tigeot if (te->ts == t) 131*77a6b00eSFrançois Tigeot goto found_and_done; 132*77a6b00eSFrançois Tigeot } 133*77a6b00eSFrançois Tigeot 134*77a6b00eSFrançois Tigeot te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); 135*77a6b00eSFrançois Tigeot te->ts = t; 136*77a6b00eSFrançois Tigeot SLIST_INSERT_HEAD(&tlist, te, tasklet_entries); 137*77a6b00eSFrançois Tigeot 138*77a6b00eSFrançois Tigeot found_and_done: 139*77a6b00eSFrançois Tigeot /* schedule the runner thread on the local cpu core */ 140*77a6b00eSFrançois Tigeot tasklet_pending = 1; 141*77a6b00eSFrançois Tigeot wakeup(&tasklet_runner); 142*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_RELEASE); 143*77a6b00eSFrançois Tigeot } 144*77a6b00eSFrançois Tigeot 145*77a6b00eSFrançois Tigeot void 146*77a6b00eSFrançois Tigeot tasklet_hi_schedule(struct tasklet_struct *t) 147*77a6b00eSFrançois Tigeot { 148*77a6b00eSFrançois Tigeot struct tasklet_entry *te; 149*77a6b00eSFrançois Tigeot 150*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_EXCLUSIVE); 151*77a6b00eSFrançois Tigeot set_bit(TASKLET_STATE_SCHED, &t->state); 152*77a6b00eSFrançois Tigeot 153*77a6b00eSFrançois Tigeot SLIST_FOREACH(te, &tlist_hi, tasklet_entries) { 154*77a6b00eSFrançois Tigeot if (te->ts == t) 155*77a6b00eSFrançois Tigeot goto found_and_done; 156*77a6b00eSFrançois Tigeot } 157*77a6b00eSFrançois Tigeot 158*77a6b00eSFrançois Tigeot te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); 159*77a6b00eSFrançois Tigeot te->ts = t; 160*77a6b00eSFrançois Tigeot SLIST_INSERT_HEAD(&tlist_hi, te, tasklet_entries); 161*77a6b00eSFrançois Tigeot 162*77a6b00eSFrançois Tigeot found_and_done: 163*77a6b00eSFrançois Tigeot /* schedule the runner thread on the local cpu core */ 164*77a6b00eSFrançois Tigeot tasklet_pending = 1; 165*77a6b00eSFrançois Tigeot wakeup(&tasklet_runner); 166*77a6b00eSFrançois Tigeot lockmgr(&tasklet_lock, LK_RELEASE); 167*77a6b00eSFrançois Tigeot } 168*77a6b00eSFrançois Tigeot 169*77a6b00eSFrançois Tigeot void 170*77a6b00eSFrançois Tigeot tasklet_kill(struct tasklet_struct *t) 171*77a6b00eSFrançois Tigeot { 172*77a6b00eSFrançois Tigeot set_bit(TASKLET_IS_DYING, &t->state); 173*77a6b00eSFrançois Tigeot wakeup(&tasklet_runner); 174*77a6b00eSFrançois Tigeot } 175*77a6b00eSFrançois Tigeot 176*77a6b00eSFrançois Tigeot static int init_tasklets(void *arg) 177*77a6b00eSFrançois Tigeot { 178*77a6b00eSFrançois Tigeot kthread_create(tasklet_runner, NULL, &tasklet_td, "tasklet_runner"); 179*77a6b00eSFrançois Tigeot 180*77a6b00eSFrançois Tigeot return 0; 181*77a6b00eSFrançois Tigeot } 182*77a6b00eSFrançois Tigeot 183*77a6b00eSFrançois Tigeot SYSINIT(linux_tasklet_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_tasklets, NULL); 184