1#!/usr/bin/env bash 2 3testdir=$(readlink -f "$(dirname "$0")") 4rootdir=$(readlink -f "$testdir/../../") 5 6source "$rootdir/test/common/autotest_common.sh" 7source "$testdir/common.sh" 8 9trap 'killprocess "$spdk_pid"' EXIT 10 11fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv")) 12# Normalize the indexes 13cpus=("${cpus[@]}") 14 15busy() { 16 local selected_cpus cpu 17 local reactor_framework 18 local threads thread 19 local sched_period=1 # default, 1s 20 21 # Create two busy threads with two cpus (not including main cpu) and check if either of 22 # them is moved to either of the selected cpus. Expected load is ~100% on each thread and 23 # each thread should remain on its designated cpu. 24 25 fold_list_onto_array selected_cpus "${cpus[@]:1:2}" 26 27 thread0=$(create_thread -n "thread0" -m "$(mask_cpus "${selected_cpus[@]}")" -a 100) 28 thread1=$(create_thread -n "thread1" -m "$(mask_cpus "${selected_cpus[@]}")" -a 100) 29 30 sleep $((10 * sched_period)) 31 32 local samples=0 33 34 xtrace_disable 35 while ((samples++ < 5)); do 36 sleep $sched_period 37 38 all_set=0 39 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 40 41 printf '*Sample %u\n' "$samples" 42 for cpu in "${selected_cpus[@]}"; do 43 threads=($(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework")) 44 45 if ((${#threads[@]} == 0)); then 46 printf ' No threads found on cpu%u\n' "$cpu" 47 continue 48 fi 49 50 get_thread_stats 51 52 for thread in "${threads[@]}"; do 53 if ((thread != thread0 && thread != thread1)); then 54 printf ' Unexpected thread %u (%s) on cpu%u\n' \ 55 "$thread" "${thread_map[thread]}" "$cpu" 56 continue 3 57 fi 58 load=$((busy[thread] * 100 / (busy[thread] + idle[thread]))) 59 if ((load < 95)); then 60 printf ' Unexpected load on thread %u (%s): %u%% (< 95%%)\n' \ 61 "$thread" "${thread_map[thread]}" "$load" 62 continue 3 63 fi 64 printf ' Thread %u (%s) on cpu%u; load: %u%%\n' \ 65 "$thread" "${thread_map[thread]}" "$cpu" "$load" 66 eval "${thread_map[thread]}_cpus[$cpu]=$cpu" 67 done 68 done 69 70 all_set=1 71 done 72 73 destroy_thread "$thread0" 74 destroy_thread "$thread1" 75 76 # The final expectation is that when target threads are ~100% busy, they will stay on their 77 # designated cpus. FIXME: Does it make sense? if given cpu is not getting a break due to a 78 # thread not becoming idle even for a tick, scheduler should not put any other threads on 79 # that cpu nor move its assigned thread to any other cpu. 80 printf 'Thread %u (%s) cpus: %s\n' "$thread0" "${thread_map[thread0]}" "${thread0_cpus[*]:-none}" 81 printf 'Thread %u (%s) cpus: %s\n' "$thread1" "${thread_map[thread1]}" "${thread1_cpus[*]:-none}" 82 [[ ${thread0_cpus[*]} != "${thread1_cpus[*]}" ]] 83 ((${#thread0_cpus[@]} == 1 && ${#thread1_cpus[@]} == 1 && all_set == 1)) 84 85 xtrace_restore 86} 87 88balanced() { 89 90 local thread cpu 91 local extra_threads 92 local sched_period=1 # default, 1s 93 local active_cpu 94 95 # Exclude main cpu 96 fold_list_onto_array selected_cpus "${cpus[@]:1}" 97 98 thread0=$(create_thread -n "thread0" -m "$(mask_cpus "${selected_cpus[@]}")" -a 0) 99 for cpu in "${selected_cpus[@]::${#selected_cpus[@]}-1}"; do 100 extra_threads+=("$(create_thread -n "thread_cpu_$cpu" -m "$(mask_cpus "$cpu")" -a 100)") 101 done 102 103 # thread0 is idle, wait for scheduler to run (2x scheduling period) and check if it is on main core 104 sleep $((2 * sched_period)) 105 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 106 [[ -n $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 107 108 # thread0 is active, wait for scheduler to run (2x) and check if it is not on main core 109 active_thread "$thread0" 100 110 sleep $((2 * sched_period)) 111 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 112 113 [[ -z $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 114 # Get the cpu thread was scheduled onto 115 for cpu in "${selected_cpus[@]}"; do 116 [[ -n $(jq -r "select(.lcore == $cpu) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" && active_cpu=$cpu 117 done 118 [[ -n ${selected_cpus[active_cpu]} ]] 119 120 # thread0 is idle, wait for scheduler to run (2x) and check if it is on main core 121 active_thread "$thread0" 0 122 sleep $((2 * sched_period)) 123 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 124 125 [[ -n $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 126 127 # thread0 is active, wait for scheduler to run (2x) and check if it is not on main core 128 active_thread "$thread0" 100 129 sleep $((2 * sched_period)) 130 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 131 132 [[ -z $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 133 134 destroy_thread "$thread0" 135 for thread in "${extra_threads[@]}"; do 136 destroy_thread "$thread" 137 done 138} 139 140core_load() { 141 local sched_period=1 # default, 1s 142 local thread 143 local on_main_core=0 on_next_core=0 144 145 # Re-exec the scheduler app to make sure rr balancer won't affect threads without 146 # configured cpumask from the previous test suites. 147 148 exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpumask" --main-core "$spdk_main_core" 149 150 # Create thread0 with 90% activity no cpumask, expecting it to remain on main cpu 151 thread0=$(create_thread -n "thread0" -a 90) 152 153 sleep $((2 * sched_period)) 154 update_thread_cpus_map 155 156 ((thread_cpus[thread0] == spdk_main_core)) 157 158 # Create thread1 with 90% activity. Expecting one of the threads to be moved to next 159 # cpu and the other remain on main cpu. Verifying that threads are spread out when core 160 # load is over 95% limit. 161 thread1=$(create_thread -n "thread1" -a 90) 162 163 # Three iterations are needed, as both active threads first are moved out of main core. 164 # During next scheduling period one of them is moved back to the main core. 165 sleep $((3 * sched_period)) 166 update_thread_cpus_map 167 168 ((thread_cpus[thread0] == spdk_main_core || thread_cpus[thread1] == spdk_main_core)) 169 ((thread_cpus[thread0] != thread_cpus[thread1])) 170 171 # Create thread2 with 10% activity. Expecting the idle thread2 to be placed on main cpu and two 172 # other active threads on next cpus. Verifying the condition where core load over 95% moves threads 173 # away from main cpu. 174 thread2=$(create_thread -n "thread2" -a 10) 175 176 sleep $((2 * sched_period)) 177 update_thread_cpus_map 178 179 ((thread_cpus[thread2] == spdk_main_core)) 180 ((thread_cpus[thread1] != spdk_main_core)) 181 ((thread_cpus[thread0] != spdk_main_core)) 182 ((thread_cpus[thread0] != thread_cpus[thread1])) 183 184 # Change all threads activity to 10%. Expecting all threads to be placed on main cpu. 185 # Verifying the condition where core load less than 95% is grouping multiple threads. 186 active_thread "$thread0" 10 187 active_thread "$thread1" 10 188 active_thread "$thread2" 10 189 190 sleep $((2 * sched_period)) 191 update_thread_cpus_map 192 193 for thread in \ 194 "$thread0" \ 195 "$thread1" \ 196 "$thread2"; do 197 ((thread_cpus[thread] == spdk_main_core)) 198 done 199 200 # Create thread3, thread4 and thread 5 with 25% activity. Expecting one of the threads on next cpu 201 # and rest on main cpu. Total load on main cpu will be (10*3+25*2) 80%, and next cpu 25%. 202 thread3=$(create_thread -n "thread3" -a 25) 203 thread4=$(create_thread -n "thread4" -a 25) 204 thread5=$(create_thread -n "thread5" -a 25) 205 206 # Three iterations are needed, as all threads look active on first iteration since they are on the main core. 207 # Second iteration will have them spread out over cores and only third will collapse to the expected scenario. 208 sleep $((3 * sched_period)) 209 update_thread_cpus_map 210 211 # Verify that load is not exceeding 80% on each of the cpus except the main and next cpu 212 get_cpu_time 5 user "${cpus[@]:2}" 213 214 for cpu in "${!avg_cpu_time[@]}"; do 215 printf '* cpu%u avg load: %u%% (%s)\n' \ 216 "$cpu" "${avg_cpu_time[cpu]}" "${cpu_times[cpu]}" 217 ((avg_cpu_time[cpu] <= 80)) 218 done 219 220 for thread in \ 221 "$thread0" \ 222 "$thread1" \ 223 "$thread2" \ 224 "$thread3" \ 225 "$thread4" \ 226 "$thread5"; do 227 if ((thread_cpus[thread] == spdk_main_core)); then 228 ((++on_main_core)) 229 else 230 ((++on_next_core)) 231 fi 232 233 destroy_thread "$thread" 234 done 235 236 ((on_main_core == 5 && on_next_core == 1)) 237} 238 239exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpumask" --main-core "$spdk_main_core" 240 241run_test "busy" busy 242run_test "balanced" balanced 243run_test "core_load" core_load 244