1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/log.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/scheduler.h" 11 12 #include "spdk_internal/event.h" 13 14 #include <rte_common.h> 15 #include <rte_version.h> 16 #if RTE_VERSION >= RTE_VERSION_NUM(24, 11, 0, 0) 17 #include <rte_power_cpufreq.h> 18 #else 19 #include <rte_power.h> 20 #endif 21 22 static uint32_t 23 _get_core_avail_freqs(uint32_t lcore_id, uint32_t *freqs, uint32_t num) 24 { 25 uint32_t rc; 26 27 rc = rte_power_freqs(lcore_id, freqs, num); 28 if (!rc) { 29 SPDK_ERRLOG("Unable to get current core frequency array for core %d\n.", lcore_id); 30 31 return 0; 32 } 33 34 return rc; 35 } 36 37 static uint32_t 38 _get_core_curr_freq(uint32_t lcore_id) 39 { 40 uint32_t freqs[SPDK_MAX_LCORE_FREQS]; 41 uint32_t freq_index; 42 int rc; 43 44 rc = rte_power_freqs(lcore_id, freqs, SPDK_MAX_LCORE_FREQS); 45 if (!rc) { 46 SPDK_ERRLOG("Unable to get current core frequency array for core %d\n.", lcore_id); 47 48 return 0; 49 } 50 freq_index = rte_power_get_freq(lcore_id); 51 if (freq_index >= SPDK_MAX_LCORE_FREQS) { 52 SPDK_ERRLOG("Unable to get current core frequency for core %d\n.", lcore_id); 53 54 return 0; 55 } 56 57 return freqs[freq_index]; 58 } 59 60 static int 61 _core_freq_up(uint32_t lcore_id) 62 { 63 return rte_power_freq_up(lcore_id); 64 } 65 66 static int 67 _core_freq_down(uint32_t lcore_id) 68 { 69 return rte_power_freq_down(lcore_id); 70 } 71 72 static int 73 _set_core_freq_max(uint32_t lcore_id) 74 { 75 return rte_power_freq_max(lcore_id); 76 } 77 78 static int 79 _set_core_freq_min(uint32_t lcore_id) 80 { 81 return rte_power_freq_min(lcore_id); 82 } 83 84 static int 85 _get_core_capabilities(uint32_t lcore_id, struct spdk_governor_capabilities *capabilities) 86 { 87 struct rte_power_core_capabilities caps; 88 int rc; 89 90 rc = rte_power_get_capabilities(lcore_id, &caps); 91 if (rc != 0) { 92 return rc; 93 } 94 95 capabilities->priority = caps.priority == 0 ? false : true; 96 97 return 0; 98 } 99 100 static int 101 _dump_info_json(struct spdk_json_write_ctx *w) 102 { 103 enum power_management_env env; 104 105 env = rte_power_get_env(); 106 107 if (env == PM_ENV_ACPI_CPUFREQ) { 108 spdk_json_write_named_string(w, "env", "acpi-cpufreq"); 109 } else if (env == PM_ENV_KVM_VM) { 110 spdk_json_write_named_string(w, "env", "kvm"); 111 } else if (env == PM_ENV_PSTATE_CPUFREQ) { 112 spdk_json_write_named_string(w, "env", "intel-pstate"); 113 } else if (env == PM_ENV_CPPC_CPUFREQ) { 114 spdk_json_write_named_string(w, "env", "cppc-cpufreq"); 115 #if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0) 116 } else if (env == PM_ENV_AMD_PSTATE_CPUFREQ) { 117 spdk_json_write_named_string(w, "env", "amd-pstate"); 118 #endif 119 } else { 120 spdk_json_write_named_string(w, "env", "unknown"); 121 return -EINVAL; 122 } 123 124 return 0; 125 } 126 127 static int 128 _init_core(uint32_t lcore_id) 129 { 130 struct rte_power_core_capabilities caps; 131 int rc; 132 133 rc = rte_power_init(lcore_id); 134 if (rc != 0) { 135 SPDK_ERRLOG("Failed to initialize on core%d\n", lcore_id); 136 return rc; 137 } 138 139 rc = rte_power_get_capabilities(lcore_id, &caps); 140 if (rc != 0) { 141 SPDK_ERRLOG("Failed retrieve capabilities of core%d\n", lcore_id); 142 return rc; 143 } 144 145 if (caps.turbo) { 146 rc = rte_power_freq_enable_turbo(lcore_id); 147 if (rc != 0) { 148 SPDK_ERRLOG("Failed to set turbo on core%d\n", lcore_id); 149 return rc; 150 } 151 } 152 153 return rc; 154 } 155 156 static int 157 _init(void) 158 { 159 struct spdk_cpuset smt_mask, app_mask; 160 uint32_t i, j; 161 int rc = 0; 162 163 if (!spdk_env_core_get_smt_cpuset(&smt_mask, UINT32_MAX)) { 164 /* We could not get SMT status on this system, don't allow 165 * the governor to load since we cannot guarantee we are running 166 * on a subset of some SMT siblings. 167 */ 168 SPDK_ERRLOG("Cannot detect SMT status\n"); 169 return -1; 170 } 171 172 /* Verify that if our app mask includes any SMT siblings, that it has 173 * all of those siblings. Otherwise the governor cannot work. 174 */ 175 spdk_env_get_cpuset(&app_mask); 176 spdk_cpuset_and(&app_mask, &smt_mask); 177 if (!spdk_cpuset_equal(&app_mask, &smt_mask)) { 178 SPDK_ERRLOG("App core mask contains some but not all of a set of SMT siblings\n"); 179 return -1; 180 } 181 182 #if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0) 183 for (i = PM_ENV_ACPI_CPUFREQ; i <= PM_ENV_AMD_PSTATE_CPUFREQ; i++) { 184 #else 185 for (i = PM_ENV_ACPI_CPUFREQ; i <= PM_ENV_CPPC_CPUFREQ; i++) { 186 #endif 187 if (rte_power_check_env_supported(i) == 1) { 188 rte_power_set_env(i); 189 break; 190 } 191 } 192 193 SPDK_ENV_FOREACH_CORE(i) { 194 rc = _init_core(i); 195 if (rc != 0) { 196 SPDK_ERRLOG("Failed to initialize on core%d\n", i); 197 break; 198 } 199 } 200 201 if (rc == 0) { 202 return rc; 203 } 204 205 /* When initialization of a core failed, deinitialize prior cores. */ 206 SPDK_ENV_FOREACH_CORE(j) { 207 if (j >= i) { 208 break; 209 } 210 if (rte_power_exit(j) != 0) { 211 SPDK_ERRLOG("Failed to deinitialize on core%d\n", j); 212 } 213 } 214 rte_power_unset_env(); 215 return rc; 216 } 217 218 static void 219 _deinit(void) 220 { 221 uint32_t i; 222 223 SPDK_ENV_FOREACH_CORE(i) { 224 if (rte_power_exit(i) != 0) { 225 SPDK_ERRLOG("Failed to deinitialize on core%d\n", i); 226 } 227 } 228 rte_power_unset_env(); 229 } 230 231 static struct spdk_governor dpdk_governor = { 232 .name = "dpdk_governor", 233 .get_core_avail_freqs = _get_core_avail_freqs, 234 .get_core_curr_freq = _get_core_curr_freq, 235 .core_freq_up = _core_freq_up, 236 .core_freq_down = _core_freq_down, 237 .set_core_freq_max = _set_core_freq_max, 238 .set_core_freq_min = _set_core_freq_min, 239 .get_core_capabilities = _get_core_capabilities, 240 .dump_info_json = _dump_info_json, 241 .init = _init, 242 .deinit = _deinit, 243 }; 244 245 SPDK_GOVERNOR_REGISTER(dpdk_governor); 246