1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef SPDK_INTERNAL_EVENT_H 35 #define SPDK_INTERNAL_EVENT_H 36 37 #include "spdk/stdinc.h" 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include "spdk/event.h" 44 #include "spdk/json.h" 45 #include "spdk/thread.h" 46 #include "spdk/util.h" 47 48 struct spdk_event { 49 uint32_t lcore; 50 spdk_event_fn fn; 51 void *arg1; 52 void *arg2; 53 }; 54 55 enum spdk_reactor_state { 56 SPDK_REACTOR_STATE_UNINITIALIZED = 0, 57 SPDK_REACTOR_STATE_INITIALIZED = 1, 58 SPDK_REACTOR_STATE_RUNNING = 2, 59 SPDK_REACTOR_STATE_EXITING = 3, 60 SPDK_REACTOR_STATE_SHUTDOWN = 4, 61 }; 62 63 struct spdk_lw_thread { 64 TAILQ_ENTRY(spdk_lw_thread) link; 65 uint64_t tsc_start; 66 uint32_t lcore; 67 bool resched; 68 /* stats over a lifetime of a thread */ 69 struct spdk_thread_stats total_stats; 70 /* stats during the last scheduling period */ 71 struct spdk_thread_stats current_stats; 72 }; 73 74 /** 75 * Completion callback to set reactor into interrupt mode or poll mode. 76 * 77 * \param cb_arg Argument to pass to the callback function. 78 */ 79 typedef void (*spdk_reactor_set_interrupt_mode_cb)(void *cb_arg); 80 81 struct spdk_reactor { 82 /* Lightweight threads running on this reactor */ 83 TAILQ_HEAD(, spdk_lw_thread) threads; 84 uint32_t thread_count; 85 86 /* Logical core number for this reactor. */ 87 uint32_t lcore; 88 89 struct { 90 uint32_t is_valid : 1; 91 uint32_t is_scheduling : 1; 92 uint32_t reserved : 30; 93 } flags; 94 95 uint64_t tsc_last; 96 97 struct spdk_ring *events; 98 int events_fd; 99 100 /* The last known rusage values */ 101 struct rusage rusage; 102 uint64_t last_rusage; 103 104 uint64_t busy_tsc; 105 uint64_t idle_tsc; 106 107 /* Each bit of cpuset indicates whether a reactor probably requires event notification */ 108 struct spdk_cpuset notify_cpuset; 109 /* Indicate whether this reactor currently runs in interrupt */ 110 bool in_interrupt; 111 bool set_interrupt_mode_in_progress; 112 bool new_in_interrupt; 113 spdk_reactor_set_interrupt_mode_cb set_interrupt_mode_cb_fn; 114 void *set_interrupt_mode_cb_arg; 115 116 struct spdk_fd_group *fgrp; 117 int resched_fd; 118 } __attribute__((aligned(SPDK_CACHE_LINE_SIZE))); 119 120 int spdk_reactors_init(void); 121 void spdk_reactors_fini(void); 122 123 void spdk_reactors_start(void); 124 void spdk_reactors_stop(void *arg1); 125 126 struct spdk_reactor *spdk_reactor_get(uint32_t lcore); 127 128 struct spdk_reactor *_spdk_get_scheduling_reactor(void); 129 130 /** 131 * Allocate and pass an event to each reactor, serially. 132 * 133 * The allocated event is processed asynchronously - i.e. spdk_for_each_reactor 134 * will return prior to `fn` being called on each reactor. 135 * 136 * \param fn This is the function that will be called on each reactor. 137 * \param arg1 Argument will be passed to fn when called. 138 * \param arg2 Argument will be passed to fn when called. 139 * \param cpl This will be called on the originating reactor after `fn` has been 140 * called on each reactor. 141 */ 142 void spdk_for_each_reactor(spdk_event_fn fn, void *arg1, void *arg2, spdk_event_fn cpl); 143 144 /** 145 * Set reactor into interrupt mode or back to poll mode. 146 * 147 * Currently, this function is only permitted within spdk application thread. 148 * Also it requires the corresponding reactor does not have any spdk_thread. 149 * 150 * \param lcore CPU core index of specified reactor. 151 * \param new_in_interrupt Set interrupt mode for true, or poll mode for false. 152 * \param cb_fn This will be called on spdk application thread after setting interupt mode. 153 * \param cb_arg Argument will be passed to cb_fn when called. 154 * 155 * \return 0 on success, negtive errno on failure. 156 */ 157 int spdk_reactor_set_interrupt_mode(uint32_t lcore, bool new_in_interrupt, 158 spdk_reactor_set_interrupt_mode_cb cb_fn, void *cb_arg); 159 160 /** 161 * Get a handle to spdk application thread. 162 * 163 * \return a pointer to spdk application thread on success or NULL on failure. 164 */ 165 struct spdk_thread *_spdk_get_app_thread(void); 166 167 struct spdk_governor_capabilities { 168 bool freq_change; 169 bool freq_getset; 170 bool freq_up; 171 bool freq_down; 172 bool freq_max; 173 bool freq_min; 174 bool turbo_set; 175 bool turbo_available; 176 bool priority; 177 }; 178 179 /** Cores governor */ 180 struct spdk_governor { 181 char *name; 182 183 /* freqs - the buffer array to save the frequencies; num - the number of frequencies to get; return - the number of available frequencies */ 184 uint32_t (*get_core_freqs)(uint32_t lcore_id, uint32_t *freqs, uint32_t num); 185 186 /* return - current frequency on success, 0 on failure */ 187 uint32_t (*get_core_curr_freq)(uint32_t lcore_id); 188 189 /** 190 * freq_index - index of available frequencies returned from get_core_freqs call 191 * 192 * return 193 * - 1 on success with frequency changed. 194 * - 0 on success without frequency changed. 195 * - Negative on error. 196 */ 197 int (*set_core_freq)(uint32_t lcore_id, uint32_t freq_index); 198 int (*core_freq_up)(uint32_t lcore_id); 199 int (*core_freq_down)(uint32_t lcore_id); 200 int (*set_core_freq_max)(uint32_t lcore_id); 201 int (*set_core_freq_min)(uint32_t lcore_id); 202 203 /** 204 * return 205 * - 1 Turbo Boost is enabled for this lcore. 206 * - 0 Turbo Boost is disabled for this lcore. 207 * - Negative on error. 208 */ 209 int (*get_core_turbo_status)(uint32_t lcore_id); 210 211 /* return - 0 on success; negative on error */ 212 int (*enable_core_turbo)(uint32_t lcore_id); 213 int (*disable_core_turbo)(uint32_t lcore_id); 214 int (*get_core_capabilities)(uint32_t lcore_id, struct spdk_governor_capabilities *capabilities); 215 int (*init_core)(uint32_t lcore_id); 216 int (*deinit_core)(uint32_t lcore_id); 217 int (*init)(void); 218 int (*deinit)(void); 219 220 TAILQ_ENTRY(spdk_governor) link; 221 }; 222 223 /** 224 * Add the given governor to the list of registered governors. 225 * This function should be invoked by referencing the macro 226 * SPDK_GOVERNOR_REGISTER in the governor c file. 227 * 228 * \param governor Governor to be added. 229 * 230 * \return 0 on success or non-zero on failure. 231 */ 232 void _spdk_governor_list_add(struct spdk_governor *governor); 233 234 /** 235 * Change current governor. 236 * 237 * \param name Name of the governor to be used. 238 * 239 * \return 0 on success or non-zero on failure. 240 */ 241 int _spdk_governor_set(char *name); 242 243 /** 244 * Get currently set governor. 245 * 246 */ 247 struct spdk_governor *_spdk_governor_get(void); 248 249 /** 250 * Macro used to register new cores governor. 251 */ 252 #define SPDK_GOVERNOR_REGISTER(governor) \ 253 static void __attribute__((constructor)) _spdk_governor_register_##name(void) \ 254 { \ 255 _spdk_governor_list_add(governor); \ 256 } \ 257 258 /** 259 * A list of cores and threads which is used for scheduling. 260 */ 261 struct spdk_scheduler_core_info { 262 /* stats over a lifetime of a core */ 263 uint64_t total_idle_tsc; 264 uint64_t total_busy_tsc; 265 /* stats during the last scheduling period */ 266 uint64_t current_idle_tsc; 267 uint64_t current_busy_tsc; 268 269 uint32_t lcore; 270 uint32_t threads_count; 271 bool interrupt_mode; 272 struct spdk_lw_thread **threads; 273 }; 274 275 /** 276 * Scheduler balance function type. 277 * Accepts array of core_info which is of size 'count' and returns updated array. 278 */ 279 typedef void (*spdk_scheduler_balance_fn)(struct spdk_scheduler_core_info *core_info, int count, 280 struct spdk_governor *governor); 281 282 /** 283 * Scheduler init function type. 284 * Called on scheduler module initialization. 285 */ 286 typedef int (*spdk_scheduler_init_fn)(struct spdk_governor *governor); 287 288 /** 289 * Scheduler deinitialization function type. 290 * Called on reactor fini. 291 */ 292 typedef int (*spdk_scheduler_deinit_fn)(struct spdk_governor *governor); 293 294 /** Thread scheduler */ 295 struct spdk_scheduler { 296 char *name; 297 spdk_scheduler_init_fn init; 298 spdk_scheduler_deinit_fn deinit; 299 spdk_scheduler_balance_fn balance; 300 TAILQ_ENTRY(spdk_scheduler) link; 301 }; 302 303 /** 304 * Add the given scheduler to the list of registered schedulers. 305 * This function should be invoked by referencing the macro 306 * SPDK_SCHEDULER_REGISTER in the scheduler c file. 307 * 308 * \param scheduler Scheduler to be added. 309 */ 310 void _spdk_scheduler_list_add(struct spdk_scheduler *scheduler); 311 312 /** 313 * Change current scheduler. 314 * 315 * \param name Name of the scheduler to be used. 316 * 317 * \return 0 on success or non-zero on failure. 318 */ 319 int _spdk_scheduler_set(char *name); 320 321 /** 322 * Get currently set scheduler. 323 * 324 */ 325 struct spdk_scheduler *_spdk_scheduler_get(void); 326 327 /** 328 * Change current scheduling period. 329 * 330 * \param period New period (microseconds). 331 */ 332 void _spdk_scheduler_period_set(uint64_t period); 333 334 /** 335 * Disable the scheduler. 336 */ 337 void _spdk_scheduler_disable(void); 338 339 /** 340 * Get period of currently set scheduler. 341 */ 342 uint64_t _spdk_scheduler_period_get(void); 343 344 /* 345 * Macro used to register new reactor balancer. 346 */ 347 #define SPDK_SCHEDULER_REGISTER(scheduler) \ 348 static void __attribute__((constructor)) _spdk_scheduler_register_ ## scheduler (void) \ 349 { \ 350 _spdk_scheduler_list_add(&scheduler); \ 351 } \ 352 353 /** 354 * Set new CPU core index. Used for scheduling, assigns new CPU core index and marks it = 355 * for rescheduling - does not actually change it. Can be used with SPDK_ENV_LCORE_ID_ANY 356 * 357 * \param thread thread to change core. 358 * \param lcore new CPU core index. 359 */ 360 void _spdk_lw_thread_set_core(struct spdk_lw_thread *thread, uint32_t lcore); 361 362 #ifdef __cplusplus 363 } 364 #endif 365 366 #endif /* SPDK_INTERNAL_EVENT_H */ 367