1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/log.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/scheduler.h" 11 12 #include "spdk_internal/event.h" 13 14 #include <rte_power.h> 15 #include <rte_version.h> 16 17 static uint32_t 18 _get_core_avail_freqs(uint32_t lcore_id, uint32_t *freqs, uint32_t num) 19 { 20 uint32_t rc; 21 22 rc = rte_power_freqs(lcore_id, freqs, num); 23 if (!rc) { 24 SPDK_ERRLOG("Unable to get current core frequency array for core %d\n.", lcore_id); 25 26 return 0; 27 } 28 29 return rc; 30 } 31 32 static uint32_t 33 _get_core_curr_freq(uint32_t lcore_id) 34 { 35 uint32_t freqs[SPDK_MAX_LCORE_FREQS]; 36 uint32_t freq_index; 37 int rc; 38 39 rc = rte_power_freqs(lcore_id, freqs, SPDK_MAX_LCORE_FREQS); 40 if (!rc) { 41 SPDK_ERRLOG("Unable to get current core frequency array for core %d\n.", lcore_id); 42 43 return 0; 44 } 45 freq_index = rte_power_get_freq(lcore_id); 46 if (freq_index >= SPDK_MAX_LCORE_FREQS) { 47 SPDK_ERRLOG("Unable to get current core frequency for core %d\n.", lcore_id); 48 49 return 0; 50 } 51 52 return freqs[freq_index]; 53 } 54 55 static int 56 _core_freq_up(uint32_t lcore_id) 57 { 58 return rte_power_freq_up(lcore_id); 59 } 60 61 static int 62 _core_freq_down(uint32_t lcore_id) 63 { 64 return rte_power_freq_down(lcore_id); 65 } 66 67 static int 68 _set_core_freq_max(uint32_t lcore_id) 69 { 70 return rte_power_freq_max(lcore_id); 71 } 72 73 static int 74 _set_core_freq_min(uint32_t lcore_id) 75 { 76 return rte_power_freq_min(lcore_id); 77 } 78 79 static int 80 _get_core_capabilities(uint32_t lcore_id, struct spdk_governor_capabilities *capabilities) 81 { 82 struct rte_power_core_capabilities caps; 83 int rc; 84 85 rc = rte_power_get_capabilities(lcore_id, &caps); 86 if (rc != 0) { 87 return rc; 88 } 89 90 capabilities->priority = caps.priority == 0 ? false : true; 91 92 return 0; 93 } 94 95 static int 96 _dump_info_json(struct spdk_json_write_ctx *w) 97 { 98 enum power_management_env env; 99 100 env = rte_power_get_env(); 101 102 if (env == PM_ENV_ACPI_CPUFREQ) { 103 spdk_json_write_named_string(w, "env", "acpi-cpufreq"); 104 } else if (env == PM_ENV_KVM_VM) { 105 spdk_json_write_named_string(w, "env", "kvm"); 106 } else if (env == PM_ENV_PSTATE_CPUFREQ) { 107 spdk_json_write_named_string(w, "env", "intel-pstate"); 108 } else if (env == PM_ENV_CPPC_CPUFREQ) { 109 spdk_json_write_named_string(w, "env", "cppc-cpufreq"); 110 #if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0) 111 } else if (env == PM_ENV_AMD_PSTATE_CPUFREQ) { 112 spdk_json_write_named_string(w, "env", "amd-pstate"); 113 #endif 114 } else { 115 spdk_json_write_named_string(w, "env", "unknown"); 116 return -EINVAL; 117 } 118 119 return 0; 120 } 121 122 static int 123 _init_core(uint32_t lcore_id) 124 { 125 struct rte_power_core_capabilities caps; 126 int rc; 127 128 rc = rte_power_init(lcore_id); 129 if (rc != 0) { 130 SPDK_ERRLOG("Failed to initialize on core%d\n", lcore_id); 131 return rc; 132 } 133 134 rc = rte_power_get_capabilities(lcore_id, &caps); 135 if (rc != 0) { 136 SPDK_ERRLOG("Failed retrieve capabilities of core%d\n", lcore_id); 137 return rc; 138 } 139 140 if (caps.turbo) { 141 rc = rte_power_freq_enable_turbo(lcore_id); 142 if (rc != 0) { 143 SPDK_ERRLOG("Failed to set turbo on core%d\n", lcore_id); 144 return rc; 145 } 146 } 147 148 return rc; 149 } 150 151 static int 152 _init(void) 153 { 154 struct spdk_cpuset smt_mask, app_mask; 155 uint32_t i, j; 156 int rc = 0; 157 158 if (!spdk_env_core_get_smt_cpuset(&smt_mask, UINT32_MAX)) { 159 /* We could not get SMT status on this system, don't allow 160 * the governor to load since we cannot guarantee we are running 161 * on a subset of some SMT siblings. 162 */ 163 SPDK_ERRLOG("Cannot detect SMT status\n"); 164 return -1; 165 } 166 167 /* Verify that if our app mask includes any SMT siblings, that it has 168 * all of those siblings. Otherwise the governor cannot work. 169 */ 170 spdk_env_get_cpuset(&app_mask); 171 spdk_cpuset_and(&app_mask, &smt_mask); 172 if (!spdk_cpuset_equal(&app_mask, &smt_mask)) { 173 SPDK_ERRLOG("App core mask contains some but not all of a set of SMT siblings\n"); 174 return -1; 175 } 176 177 #if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0) 178 for (i = PM_ENV_ACPI_CPUFREQ; i <= PM_ENV_AMD_PSTATE_CPUFREQ; i++) { 179 #else 180 for (i = PM_ENV_ACPI_CPUFREQ; i <= PM_ENV_CPPC_CPUFREQ; i++) { 181 #endif 182 if (rte_power_check_env_supported(i) == 1) { 183 rte_power_set_env(i); 184 break; 185 } 186 } 187 188 SPDK_ENV_FOREACH_CORE(i) { 189 rc = _init_core(i); 190 if (rc != 0) { 191 SPDK_ERRLOG("Failed to initialize on core%d\n", i); 192 break; 193 } 194 } 195 196 if (rc == 0) { 197 return rc; 198 } 199 200 /* When initialization of a core failed, deinitialize prior cores. */ 201 SPDK_ENV_FOREACH_CORE(j) { 202 if (j >= i) { 203 break; 204 } 205 if (rte_power_exit(j) != 0) { 206 SPDK_ERRLOG("Failed to deinitialize on core%d\n", j); 207 } 208 } 209 rte_power_unset_env(); 210 return rc; 211 } 212 213 static void 214 _deinit(void) 215 { 216 uint32_t i; 217 218 SPDK_ENV_FOREACH_CORE(i) { 219 if (rte_power_exit(i) != 0) { 220 SPDK_ERRLOG("Failed to deinitialize on core%d\n", i); 221 } 222 } 223 rte_power_unset_env(); 224 } 225 226 static struct spdk_governor dpdk_governor = { 227 .name = "dpdk_governor", 228 .get_core_avail_freqs = _get_core_avail_freqs, 229 .get_core_curr_freq = _get_core_curr_freq, 230 .core_freq_up = _core_freq_up, 231 .core_freq_down = _core_freq_down, 232 .set_core_freq_max = _set_core_freq_max, 233 .set_core_freq_min = _set_core_freq_min, 234 .get_core_capabilities = _get_core_capabilities, 235 .dump_info_json = _dump_info_json, 236 .init = _init, 237 .deinit = _deinit, 238 }; 239 240 SPDK_GOVERNOR_REGISTER(dpdk_governor); 241