1 /* 2 * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <linux/interrupt.h> 28 #include <linux/slab.h> 29 30 #include <sys/kthread.h> 31 32 /* 33 * Linux tasklet constraints: 34 * - tasklets that have the same type cannot be run on multiple processors at 35 * the same time 36 * - tasklets always run on the processor from which they were originally 37 * submitted 38 * - when a tasklet is scheduled, its state is set to TASKLET_STATE_SCHED, 39 * and the tasklet added to a queue 40 * - during the execution of its function, the tasklet state is set to 41 * TASKLET_STATE_RUN and the TASKLET_STATE_SCHED state is removed 42 */ 43 44 struct tasklet_entry { 45 struct tasklet_struct *ts; 46 SLIST_ENTRY(tasklet_entry) tasklet_entries; 47 }; 48 49 static struct lock tasklet_lock = LOCK_INITIALIZER("dltll", 0, LK_CANRECURSE); 50 51 static struct thread *tasklet_td = NULL; 52 SLIST_HEAD(tasklet_list_head, tasklet_entry) tlist = SLIST_HEAD_INITIALIZER(tlist); 53 SLIST_HEAD(tasklet_hi_list_head, tasklet_entry) tlist_hi = SLIST_HEAD_INITIALIZER(tlist_hi); 54 55 static int tasklet_pending = 0; 56 57 #define PROCESS_TASKLET_LIST(which_list) do { \ 58 SLIST_FOREACH_MUTABLE(te, &which_list, tasklet_entries, tmp_te) { \ 59 struct tasklet_struct *t = te->ts; \ 60 \ 61 /* \ 62 This tasklet is dying, remove it from the list. \ 63 We allow to it to run one last time if it has \ 64 already been scheduled. \ 65 */ \ 66 if (test_bit(TASKLET_IS_DYING, &t->state)) { \ 67 SLIST_REMOVE(&which_list, te, tasklet_entry, tasklet_entries); \ 68 kfree(te); \ 69 } \ 70 \ 71 /* This tasklet is not enabled, try the next one */ \ 72 if (atomic_read(&t->count) != 0) \ 73 continue; \ 74 \ 75 /* This tasklet is not scheduled, try the next one */ \ 76 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) \ 77 continue; \ 78 \ 79 clear_bit(TASKLET_STATE_SCHED, &t->state); \ 80 set_bit(TASKLET_STATE_RUN, &t->state); \ 81 \ 82 lockmgr(&tasklet_lock, LK_RELEASE); \ 83 if (t->func) \ 84 t->func(t->data); \ 85 lockmgr(&tasklet_lock, LK_EXCLUSIVE); \ 86 \ 87 clear_bit(TASKLET_STATE_RUN, &t->state); \ 88 } \ 89 } while (0) 90 91 /* XXX runners should be CPU-specific */ 92 static void 93 tasklet_runner(void *arg) 94 { 95 struct tasklet_entry *te, *tmp_te; 96 97 lockmgr(&tasklet_lock, LK_EXCLUSIVE); 98 while (1) { 99 /* 100 Only sleep if we haven't been raced by a _schedule() 101 call during an unlock window 102 */ 103 if (tasklet_pending == 0) { 104 lksleep(&tasklet_runner, &tasklet_lock, 0, "tkidle", 0); 105 } 106 tasklet_pending = 0; 107 108 /* Process hi tasklets first */ 109 PROCESS_TASKLET_LIST(tlist_hi); 110 PROCESS_TASKLET_LIST(tlist); 111 } 112 lockmgr(&tasklet_lock, LK_RELEASE); 113 } 114 115 void 116 tasklet_init(struct tasklet_struct *t, 117 void (*func)(unsigned long), unsigned long data) 118 { 119 t->state = 0; 120 t->func = func; 121 t->data = data; 122 atomic_set(&t->count, 0); 123 } 124 125 void 126 tasklet_schedule(struct tasklet_struct *t) 127 { 128 struct tasklet_entry *te; 129 130 lockmgr(&tasklet_lock, LK_EXCLUSIVE); 131 set_bit(TASKLET_STATE_SCHED, &t->state); 132 133 SLIST_FOREACH(te, &tlist, tasklet_entries) { 134 if (te->ts == t) 135 goto found_and_done; 136 } 137 138 te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); 139 te->ts = t; 140 SLIST_INSERT_HEAD(&tlist, te, tasklet_entries); 141 142 found_and_done: 143 /* schedule the runner thread on the local cpu core */ 144 tasklet_pending = 1; 145 wakeup(&tasklet_runner); 146 lockmgr(&tasklet_lock, LK_RELEASE); 147 } 148 149 void 150 tasklet_hi_schedule(struct tasklet_struct *t) 151 { 152 struct tasklet_entry *te; 153 154 lockmgr(&tasklet_lock, LK_EXCLUSIVE); 155 set_bit(TASKLET_STATE_SCHED, &t->state); 156 157 SLIST_FOREACH(te, &tlist_hi, tasklet_entries) { 158 if (te->ts == t) 159 goto found_and_done; 160 } 161 162 te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); 163 te->ts = t; 164 SLIST_INSERT_HEAD(&tlist_hi, te, tasklet_entries); 165 166 found_and_done: 167 /* schedule the runner thread on the local cpu core */ 168 tasklet_pending = 1; 169 wakeup(&tasklet_runner); 170 lockmgr(&tasklet_lock, LK_RELEASE); 171 } 172 173 void 174 tasklet_kill(struct tasklet_struct *t) 175 { 176 set_bit(TASKLET_IS_DYING, &t->state); 177 wakeup(&tasklet_runner); 178 } 179 180 static int init_tasklets(void *arg) 181 { 182 kthread_create(tasklet_runner, NULL, &tasklet_td, "tasklet_runner"); 183 184 return 0; 185 } 186 187 SYSINIT(linux_tasklet_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_tasklets, NULL); 188