1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2021 Intel Corporation 4# All rights reserved. 5# 6testdir=$(readlink -f "$(dirname "$0")") 7rootdir=$(readlink -f "$testdir/../../") 8 9source "$rootdir/test/common/autotest_common.sh" 10source "$testdir/common.sh" 11 12trap 'killprocess "$spdk_pid"' EXIT 13 14fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv")) 15# Normalize the indexes 16cpus=("${cpus[@]}") 17 18busy() { 19 local selected_cpus cpu 20 local reactor_framework 21 local threads thread 22 local sched_period=1 # default, 1s 23 24 # Create two busy threads with two cpus (not including main cpu) and check if either of 25 # them is moved to either of the selected cpus. Expected load is ~100% on each thread and 26 # each thread should remain on its designated cpu. 27 28 fold_list_onto_array selected_cpus "${cpus[@]:1:2}" 29 30 thread0=$(create_thread -n "thread0" -m "$(mask_cpus "${selected_cpus[@]}")" -a 100) 31 thread1=$(create_thread -n "thread1" -m "$(mask_cpus "${selected_cpus[@]}")" -a 100) 32 33 sleep $((10 * sched_period)) 34 35 local samples=0 36 37 xtrace_disable 38 while ((samples++ < 5)); do 39 sleep $sched_period 40 41 all_set=0 42 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 43 44 printf '*Sample %u\n' "$samples" 45 for cpu in "${selected_cpus[@]}"; do 46 threads=($(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework")) 47 48 if ((${#threads[@]} == 0)); then 49 printf ' No threads found on cpu%u\n' "$cpu" 50 continue 51 fi 52 53 get_thread_stats 54 55 for thread in "${threads[@]}"; do 56 if ((thread != thread0 && thread != thread1)); then 57 printf ' Unexpected thread %u (%s) on cpu%u\n' \ 58 "$thread" "${thread_map[thread]}" "$cpu" 59 continue 3 60 fi 61 load=$((busy[thread] * 100 / (busy[thread] + idle[thread]))) 62 if ((load < 95)); then 63 printf ' Unexpected load on thread %u (%s): %u%% (< 95%%)\n' \ 64 "$thread" "${thread_map[thread]}" "$load" 65 continue 3 66 fi 67 printf ' Thread %u (%s) on cpu%u; load: %u%%\n' \ 68 "$thread" "${thread_map[thread]}" "$cpu" "$load" 69 eval "${thread_map[thread]}_cpus[$cpu]=$cpu" 70 done 71 done 72 73 all_set=1 74 done 75 76 destroy_thread "$thread0" 77 destroy_thread "$thread1" 78 79 # The final expectation is that when target threads are ~100% busy, they will stay on their 80 # designated cpus. FIXME: Does it make sense? if given cpu is not getting a break due to a 81 # thread not becoming idle even for a tick, scheduler should not put any other threads on 82 # that cpu nor move its assigned thread to any other cpu. 83 printf 'Thread %u (%s) cpus: %s\n' "$thread0" "${thread_map[thread0]}" "${thread0_cpus[*]:-none}" 84 printf 'Thread %u (%s) cpus: %s\n' "$thread1" "${thread_map[thread1]}" "${thread1_cpus[*]:-none}" 85 [[ ${thread0_cpus[*]} != "${thread1_cpus[*]}" ]] 86 ((${#thread0_cpus[@]} == 1 && ${#thread1_cpus[@]} == 1 && all_set == 1)) 87 88 xtrace_restore 89} 90 91balanced() { 92 93 local thread cpu 94 local extra_threads 95 local sched_period=1 # default, 1s 96 local active_cpu 97 98 # Exclude main cpu 99 fold_list_onto_array selected_cpus "${cpus[@]:1}" 100 101 thread0=$(create_thread -n "thread0" -m "$(mask_cpus "${selected_cpus[@]}")" -a 0) 102 for cpu in "${selected_cpus[@]::${#selected_cpus[@]}-1}"; do 103 extra_threads+=("$(create_thread -n "thread_cpu_$cpu" -m "$(mask_cpus "$cpu")" -a 100)") 104 done 105 106 # thread0 is idle, wait for scheduler to run (2x scheduling period) and check if it is on main core 107 sleep $((2 * sched_period)) 108 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 109 [[ -n $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 110 111 # thread0 is active, wait for scheduler to run (2x) and check if it is not on main core 112 active_thread "$thread0" 100 113 sleep $((2 * sched_period)) 114 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 115 116 [[ -z $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 117 # Get the cpu thread was scheduled onto 118 for cpu in "${selected_cpus[@]}"; do 119 [[ -n $(jq -r "select(.lcore == $cpu) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" && active_cpu=$cpu 120 done 121 [[ -n ${selected_cpus[active_cpu]} ]] 122 123 # thread0 is idle, wait for scheduler to run (2x) and check if it is on main core 124 active_thread "$thread0" 0 125 sleep $((2 * sched_period)) 126 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 127 128 [[ -n $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 129 130 # thread0 is active, wait for scheduler to run (2x) and check if it is not on main core 131 active_thread "$thread0" 100 132 sleep $((2 * sched_period)) 133 reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') 134 135 [[ -z $(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.id == $thread0)") ]] <<< "$reactor_framework" 136 137 destroy_thread "$thread0" 138 for thread in "${extra_threads[@]}"; do 139 destroy_thread "$thread" 140 done 141} 142 143core_load() { 144 local sched_period=1 # default, 1s 145 local thread 146 local on_main_core=0 on_next_core=0 147 148 # Re-exec the scheduler app to make sure rr balancer won't affect threads without 149 # configured cpumask from the previous test suites. 150 151 exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpumask" --main-core "$spdk_main_core" 152 153 # Create thread0 with 90% activity no cpumask, expecting it to remain on main cpu 154 thread0=$(create_thread -n "thread0" -a 90) 155 156 sleep $((2 * sched_period)) 157 update_thread_cpus_map 158 159 ((thread_cpus[thread0] == spdk_main_core)) 160 161 # Create thread1 with 90% activity. Expecting one of the threads to be moved to next 162 # cpu and the other remain on main cpu. Verifying that threads are spread out when core 163 # load is over 95% limit. 164 thread1=$(create_thread -n "thread1" -a 90) 165 166 # Three iterations are needed, as both active threads first are moved out of main core. 167 # During next scheduling period one of them is moved back to the main core. 168 sleep $((3 * sched_period)) 169 update_thread_cpus_map 170 171 ((thread_cpus[thread0] == spdk_main_core || thread_cpus[thread1] == spdk_main_core)) 172 ((thread_cpus[thread0] != thread_cpus[thread1])) 173 174 # Create thread2 with 10% activity. Expecting the idle thread2 to be placed on main cpu and two 175 # other active threads on next cpus. Verifying the condition where core load over 95% moves threads 176 # away from main cpu. 177 thread2=$(create_thread -n "thread2" -a 10) 178 179 sleep $((2 * sched_period)) 180 update_thread_cpus_map 181 182 ((thread_cpus[thread2] == spdk_main_core)) 183 ((thread_cpus[thread1] != spdk_main_core)) 184 ((thread_cpus[thread0] != spdk_main_core)) 185 ((thread_cpus[thread0] != thread_cpus[thread1])) 186 187 # Change all threads activity to 10%. Expecting all threads to be placed on main cpu. 188 # Verifying the condition where core load less than 95% is grouping multiple threads. 189 active_thread "$thread0" 10 190 active_thread "$thread1" 10 191 active_thread "$thread2" 10 192 193 sleep $((2 * sched_period)) 194 update_thread_cpus_map 195 196 for thread in \ 197 "$thread0" \ 198 "$thread1" \ 199 "$thread2"; do 200 ((thread_cpus[thread] == spdk_main_core)) 201 done 202 203 # Create thread3, thread4 and thread 5 with 25% activity. Expecting one of the threads on next cpu 204 # and rest on main cpu. Total load on main cpu will be (10*3+25*2) 80%, and next cpu 25%. 205 thread3=$(create_thread -n "thread3" -a 25) 206 thread4=$(create_thread -n "thread4" -a 25) 207 thread5=$(create_thread -n "thread5" -a 25) 208 209 # Three iterations are needed, as all threads look active on first iteration since they are on the main core. 210 # Second iteration will have them spread out over cores and only third will collapse to the expected scenario. 211 sleep $((3 * sched_period)) 212 update_thread_cpus_map 213 214 # Verify that load is not exceeding 80% on each of the cpus except the main and next cpu 215 get_cpu_time 5 user "${cpus[@]:2}" 216 217 for cpu in "${!avg_cpu_time[@]}"; do 218 printf '* cpu%u avg load: %u%% (%s)\n' \ 219 "$cpu" "${avg_cpu_time[cpu]}" "${cpu_times[cpu]}" 220 ((avg_cpu_time[cpu] <= 80)) 221 done 222 223 for thread in \ 224 "$thread0" \ 225 "$thread1" \ 226 "$thread2" \ 227 "$thread3" \ 228 "$thread4" \ 229 "$thread5"; do 230 if ((thread_cpus[thread] == spdk_main_core)); then 231 ((++on_main_core)) 232 else 233 ((++on_next_core)) 234 fi 235 236 destroy_thread "$thread" 237 done 238 239 ((on_main_core == 5 && on_next_core == 1)) 240} 241 242exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpumask" --main-core "$spdk_main_core" 243 244run_test "busy" busy 245run_test "balanced" balanced 246run_test "core_load" core_load 247