1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef SPDK_INTERNAL_EVENT_H 35 #define SPDK_INTERNAL_EVENT_H 36 37 #include "spdk/stdinc.h" 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include "spdk/event.h" 44 #include "spdk/json.h" 45 #include "spdk/thread.h" 46 #include "spdk/util.h" 47 48 struct spdk_event { 49 uint32_t lcore; 50 spdk_event_fn fn; 51 void *arg1; 52 void *arg2; 53 }; 54 55 enum spdk_reactor_state { 56 SPDK_REACTOR_STATE_UNINITIALIZED = 0, 57 SPDK_REACTOR_STATE_INITIALIZED = 1, 58 SPDK_REACTOR_STATE_RUNNING = 2, 59 SPDK_REACTOR_STATE_EXITING = 3, 60 SPDK_REACTOR_STATE_SHUTDOWN = 4, 61 }; 62 63 struct spdk_lw_thread { 64 TAILQ_ENTRY(spdk_lw_thread) link; 65 uint64_t tsc_start; 66 uint32_t lcore; 67 uint32_t new_lcore; 68 bool resched; 69 struct spdk_thread_stats current_stats; 70 struct spdk_thread_stats snapshot_stats; 71 }; 72 73 struct spdk_reactor { 74 /* Lightweight threads running on this reactor */ 75 TAILQ_HEAD(, spdk_lw_thread) threads; 76 uint32_t thread_count; 77 78 /* Logical core number for this reactor. */ 79 uint32_t lcore; 80 81 struct { 82 uint32_t is_valid : 1; 83 uint32_t is_scheduling : 1; 84 uint32_t reserved : 30; 85 } flags; 86 87 uint64_t tsc_last; 88 89 struct spdk_ring *events; 90 int events_fd; 91 92 /* The last known rusage values */ 93 struct rusage rusage; 94 uint64_t last_rusage; 95 96 uint64_t busy_tsc; 97 uint64_t idle_tsc; 98 99 bool interrupt_mode; 100 struct spdk_fd_group *fgrp; 101 int resched_fd; 102 } __attribute__((aligned(SPDK_CACHE_LINE_SIZE))); 103 104 int spdk_reactors_init(void); 105 void spdk_reactors_fini(void); 106 107 void spdk_reactors_start(void); 108 void spdk_reactors_stop(void *arg1); 109 110 struct spdk_reactor *spdk_reactor_get(uint32_t lcore); 111 112 /** 113 * Allocate and pass an event to each reactor, serially. 114 * 115 * The allocated event is processed asynchronously - i.e. spdk_for_each_reactor 116 * will return prior to `fn` being called on each reactor. 117 * 118 * \param fn This is the function that will be called on each reactor. 119 * \param arg1 Argument will be passed to fn when called. 120 * \param arg2 Argument will be passed to fn when called. 121 * \param cpl This will be called on the originating reactor after `fn` has been 122 * called on each reactor. 123 */ 124 void spdk_for_each_reactor(spdk_event_fn fn, void *arg1, void *arg2, spdk_event_fn cpl); 125 126 struct spdk_subsystem { 127 const char *name; 128 /* User must call spdk_subsystem_init_next() when they are done with their initialization. */ 129 void (*init)(void); 130 void (*fini)(void); 131 132 /** 133 * Write JSON configuration handler. 134 * 135 * \param w JSON write context 136 */ 137 void (*write_config_json)(struct spdk_json_write_ctx *w); 138 TAILQ_ENTRY(spdk_subsystem) tailq; 139 }; 140 141 struct spdk_subsystem *spdk_subsystem_find(const char *name); 142 struct spdk_subsystem *spdk_subsystem_get_first(void); 143 struct spdk_subsystem *spdk_subsystem_get_next(struct spdk_subsystem *cur_subsystem); 144 145 struct spdk_subsystem_depend { 146 const char *name; 147 const char *depends_on; 148 TAILQ_ENTRY(spdk_subsystem_depend) tailq; 149 }; 150 151 struct spdk_subsystem_depend *spdk_subsystem_get_first_depend(void); 152 struct spdk_subsystem_depend *spdk_subsystem_get_next_depend(struct spdk_subsystem_depend 153 *cur_depend); 154 155 void spdk_add_subsystem(struct spdk_subsystem *subsystem); 156 void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend); 157 158 typedef void (*spdk_subsystem_init_fn)(int rc, void *ctx); 159 void spdk_subsystem_init(spdk_subsystem_init_fn cb_fn, void *cb_arg); 160 void spdk_subsystem_fini(spdk_msg_fn cb_fn, void *cb_arg); 161 void spdk_subsystem_init_next(int rc); 162 void spdk_subsystem_fini_next(void); 163 void spdk_app_json_config_load(const char *json_config_file, const char *rpc_addr, 164 spdk_subsystem_init_fn cb_fn, void *cb_arg, 165 bool stop_on_error); 166 167 /** 168 * Save pointed \c subsystem configuration to the JSON write context \c w. In case of 169 * error \c null is written to the JSON context. 170 * 171 * \param w JSON write context 172 * \param subsystem the subsystem to query 173 */ 174 void spdk_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_subsystem *subsystem); 175 176 void spdk_rpc_initialize(const char *listen_addr); 177 void spdk_rpc_finish(void); 178 179 struct spdk_governor_capabilities { 180 bool freq_change; 181 bool freq_getset; 182 bool freq_up; 183 bool freq_down; 184 bool freq_max; 185 bool freq_min; 186 bool turbo_set; 187 bool turbo_available; 188 bool priority; 189 }; 190 191 /** Cores governor */ 192 struct spdk_governor { 193 char *name; 194 195 /* freqs - the buffer array to save the frequencies; num - the number of frequencies to get; return - the number of available frequencies */ 196 uint32_t (*get_core_freqs)(uint32_t lcore_id, uint32_t *freqs, uint32_t num); 197 198 /* return - current frequency */ 199 uint32_t (*get_core_curr_freq)(uint32_t lcore_id); 200 201 /** 202 * freq_index - index of available frequencies returned from get_core_freqs call 203 * 204 * return 205 * - 1 on success with frequency changed. 206 * - 0 on success without frequency changed. 207 * - Negative on error. 208 */ 209 int (*set_core_freq)(uint32_t lcore_id, uint32_t freq_index); 210 int (*core_freq_up)(uint32_t lcore_id); 211 int (*core_freq_down)(uint32_t lcore_id); 212 int (*set_core_freq_max)(uint32_t lcore_id); 213 int (*set_core_freq_min)(uint32_t lcore_id); 214 215 /** 216 * return 217 * - 1 Turbo Boost is enabled for this lcore. 218 * - 0 Turbo Boost is disabled for this lcore. 219 * - Negative on error. 220 */ 221 int (*get_core_turbo_status)(uint32_t lcore_id); 222 223 /* return - 0 on success; negative on error */ 224 int (*enable_core_turbo)(uint32_t lcore_id); 225 int (*disable_core_turbo)(uint32_t lcore_id); 226 int (*get_core_capabilities)(uint32_t lcore_id, struct spdk_governor_capabilities *capabilities); 227 int (*init_core)(uint32_t lcore_id); 228 int (*deinit_core)(uint32_t lcore_id); 229 int (*init)(void); 230 int (*deinit)(void); 231 232 TAILQ_ENTRY(spdk_governor) link; 233 }; 234 235 /** 236 * Add the given governor to the list of registered governors. 237 * This function should be invoked by referencing the macro 238 * SPDK_GOVERNOR_REGISTER in the governor c file. 239 * 240 * \param governor Governor to be added. 241 * 242 * \return 0 on success or non-zero on failure. 243 */ 244 void _spdk_governor_list_add(struct spdk_governor *governor); 245 246 /** 247 * Change current governor. 248 * 249 * \param name Name of the governor to be used. 250 * 251 * \return 0 on success or non-zero on failure. 252 */ 253 int _spdk_governor_set(char *name); 254 255 /** 256 * Macro used to register new cores governor. 257 */ 258 #define SPDK_GOVERNOR_REGISTER(governor) \ 259 static void __attribute__((constructor)) _spdk_governor_register_##name(void) \ 260 { \ 261 _spdk_governor_list_add(governor); \ 262 } \ 263 264 /** 265 * A list of cores and threads which is used for scheduling. 266 */ 267 struct spdk_scheduler_core_info { 268 uint64_t core_idle_tsc; 269 uint64_t core_busy_tsc; 270 uint32_t lcore; 271 uint32_t threads_count; 272 struct spdk_lw_thread **threads; 273 }; 274 275 /** 276 * Scheduler balance function type. 277 * Accepts array of core_info which is of size 'count' and returns updated array. 278 */ 279 typedef void (*spdk_scheduler_balance_fn)(struct spdk_scheduler_core_info *core_info, int count, 280 struct spdk_governor *governor); 281 282 /** 283 * Scheduler init function type. 284 * Called on scheduler module initialization. 285 */ 286 typedef int (*spdk_scheduler_init_fn)(struct spdk_governor *governor); 287 288 /** 289 * Scheduler deinitialization function type. 290 * Called on reactor fini. 291 */ 292 typedef int (*spdk_scheduler_deinit_fn)(struct spdk_governor *governor); 293 294 /** Thread scheduler */ 295 struct spdk_scheduler { 296 char *name; 297 spdk_scheduler_init_fn init; 298 spdk_scheduler_deinit_fn deinit; 299 spdk_scheduler_balance_fn balance; 300 TAILQ_ENTRY(spdk_scheduler) link; 301 }; 302 303 /** 304 * Add the given scheduler to the list of registered schedulers. 305 * This function should be invoked by referencing the macro 306 * SPDK_SCHEDULER_REGISTER in the scheduler c file. 307 * 308 * \param scheduler Scheduler to be added. 309 */ 310 void _spdk_scheduler_list_add(struct spdk_scheduler *scheduler); 311 312 /** 313 * Change current scheduler. 314 * 315 * \param name Name of the scheduler to be used. 316 * 317 * \return 0 on success or non-zero on failure. 318 */ 319 int _spdk_scheduler_set(char *name); 320 321 /** 322 * Change current scheduling period. 323 * 324 * \param period New period (microseconds). 325 */ 326 void _spdk_scheduler_period_set(uint64_t period); 327 328 /* 329 * Macro used to register new reactor balancer. 330 */ 331 #define SPDK_SCHEDULER_REGISTER(scheduler) \ 332 static void __attribute__((constructor)) _spdk_scheduler_register_ ## scheduler (void) \ 333 { \ 334 _spdk_scheduler_list_add(&scheduler); \ 335 } \ 336 337 /** 338 * Set new CPU core index. Used for scheduling, assigns new CPU core index and marks it = 339 * for rescheduling - does not actually change it. Can be used with SPDK_ENV_LCORE_ID_ANY 340 * 341 * \param thread thread to change core. 342 * \param lcore new CPU core index. 343 */ 344 void _spdk_lw_thread_set_core(struct spdk_lw_thread *thread, uint32_t lcore); 345 346 /** 347 * Get threads stats 348 * 349 * \param thread thread that stats regards to. 350 * \param stats Output parameter for accumulated TSC counts while the thread was busy. 351 */ 352 void _spdk_lw_thread_get_current_stats(struct spdk_lw_thread *thread, 353 struct spdk_thread_stats *stats); 354 355 /** 356 * \brief Register a new subsystem 357 */ 358 #define SPDK_SUBSYSTEM_REGISTER(_name) \ 359 __attribute__((constructor)) static void _name ## _register(void) \ 360 { \ 361 spdk_add_subsystem(&_name); \ 362 } 363 364 /** 365 * \brief Declare that a subsystem depends on another subsystem. 366 */ 367 #define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on) \ 368 static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \ 369 .name = #_name, \ 370 .depends_on = #_depends_on, \ 371 }; \ 372 __attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void) \ 373 { \ 374 spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \ 375 } 376 377 #ifdef __cplusplus 378 } 379 #endif 380 381 #endif /* SPDK_INTERNAL_EVENT_H */ 382