1#!/usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2017 Intel Corporation 4# All rights reserved. 5# 6 7# This script runs fio benchmark test on the local nvme device using the SPDK NVMe driver. 8# Prework: Run script/setup.sh to bind SSDs to SPDK driver. 9# Prework: Change any fio configurations in the template fio config file fio_test.conf 10# Output: A csv file <hostname>_<num ssds>_perf_output.csv 11 12import subprocess 13from subprocess import check_call, call, check_output, Popen, PIPE 14import random 15import os 16import sys 17import re 18import signal 19import getopt 20from datetime import datetime 21from itertools import * 22import csv 23import itertools 24from shutil import copyfile 25import json 26 27# Populate test parameters into these lists to run different workloads 28# The configuration below runs QD 1 & 128. To add QD 32 set q_depth=['1', '32', '128'] 29q_depth = ['1', '128'] 30# io_size specifies the size in bytes of the IO workload. 31# To add 64K IOs set io_size = ['4096', '65536'] 32io_size = ['4096'] 33workload_type = ['randrw'] 34mix = ['100'] 35core_mask = ['0x1'] 36# run_time parameter specifies how long to run each test. 37# Set run_time = ['600'] to run the test for 10 minutes 38run_time = ['60'] 39# iter_num parameter is used to run the test multiple times. 40# set iter_num = ['1', '2', '3'] to repeat each test 3 times 41iter_num = ['1'] 42 43 44def run_fio(io_size_bytes, qd, rw_mix, cpu_mask, run_num, workload, run_time_sec): 45 print("Running Test: IO Size={} QD={} Mix={} CPU Mask={}".format(io_size_bytes, qd, rw_mix, cpu_mask)) 46 string = "s_" + str(io_size_bytes) + "_q_" + str(qd) + "_m_" + str(rw_mix) + "_c_" + str(cpu_mask) + "_run_" + str(run_num) 47 48 # Call fio 49 path_to_fio_conf = config_file_for_test 50 path_to_ioengine = sys.argv[2] 51 command = "BLK_SIZE=" + str(io_size_bytes) + " RW=" + str(workload) + " MIX=" + str(rw_mix) \ 52 + " IODEPTH=" + str(qd) + " RUNTIME=" + str(run_time_sec) + " IOENGINE=" + path_to_ioengine \ 53 + " fio " + str(path_to_fio_conf) + " -output=" + string + " -output-format=json" 54 output = subprocess.check_output(command, shell=True) 55 56 print("Finished Test: IO Size={} QD={} Mix={} CPU Mask={}".format(io_size_bytes, qd, rw_mix, cpu_mask)) 57 return 58 59 60def parse_results(io_size_bytes, qd, rw_mix, cpu_mask, run_num, workload, run_time_sec): 61 results_array = [] 62 63 # If json file has results for multiple fio jobs pick the results from the right job 64 job_pos = 0 65 66 # generate the next result line that will be added to the output csv file 67 results = str(io_size_bytes) + "," + str(qd) + "," + str(rw_mix) + "," \ 68 + str(workload) + "," + str(cpu_mask) + "," + str(run_time_sec) + "," + str(run_num) 69 70 # Read the results of this run from the test result file 71 string = "s_" + str(io_size_bytes) + "_q_" + str(qd) + "_m_" + str(rw_mix) + "_c_" + str(cpu_mask) + "_run_" + str(run_num) 72 with open(string) as json_file: 73 data = json.load(json_file) 74 job_name = data['jobs'][job_pos]['jobname'] 75 # print "FIO job name: ", job_name 76 if 'lat_ns' in data['jobs'][job_pos]['read']: 77 lat = 'lat_ns' 78 lat_units = 'ns' 79 else: 80 lat = 'lat' 81 lat_units = 'us' 82 read_iops = float(data['jobs'][job_pos]['read']['iops']) 83 read_bw = float(data['jobs'][job_pos]['read']['bw']) 84 read_avg_lat = float(data['jobs'][job_pos]['read'][lat]['mean']) 85 read_min_lat = float(data['jobs'][job_pos]['read'][lat]['min']) 86 read_max_lat = float(data['jobs'][job_pos]['read'][lat]['max']) 87 write_iops = float(data['jobs'][job_pos]['write']['iops']) 88 write_bw = float(data['jobs'][job_pos]['write']['bw']) 89 write_avg_lat = float(data['jobs'][job_pos]['write'][lat]['mean']) 90 write_min_lat = float(data['jobs'][job_pos]['write'][lat]['min']) 91 write_max_lat = float(data['jobs'][job_pos]['write'][lat]['max']) 92 print("%-10s" % "IO Size", "%-10s" % "QD", "%-10s" % "Mix", 93 "%-10s" % "Workload Type", "%-10s" % "CPU Mask", 94 "%-10s" % "Run Time", "%-10s" % "Run Num", 95 "%-15s" % "Read IOps", 96 "%-10s" % "Read MBps", "%-15s" % "Read Avg. Lat(" + lat_units + ")", 97 "%-15s" % "Read Min. Lat(" + lat_units + ")", "%-15s" % "Read Max. Lat(" + lat_units + ")", 98 "%-15s" % "Write IOps", 99 "%-10s" % "Write MBps", "%-15s" % "Write Avg. Lat(" + lat_units + ")", 100 "%-15s" % "Write Min. Lat(" + lat_units + ")", "%-15s" % "Write Max. Lat(" + lat_units + ")") 101 print("%-10s" % io_size_bytes, "%-10s" % qd, "%-10s" % rw_mix, 102 "%-10s" % workload, "%-10s" % cpu_mask, "%-10s" % run_time_sec, 103 "%-10s" % run_num, "%-15s" % read_iops, "%-10s" % read_bw, 104 "%-15s" % read_avg_lat, "%-15s" % read_min_lat, "%-15s" % read_max_lat, 105 "%-15s" % write_iops, "%-10s" % write_bw, "%-15s" % write_avg_lat, 106 "%-15s" % write_min_lat, "%-15s" % write_max_lat) 107 results = results + "," + str(read_iops) + "," + str(read_bw) + "," \ 108 + str(read_avg_lat) + "," + str(read_min_lat) + "," + str(read_max_lat) \ 109 + "," + str(write_iops) + "," + str(write_bw) + "," + str(write_avg_lat) \ 110 + "," + str(write_min_lat) + "," + str(write_max_lat) 111 with open(result_file_name, "a") as result_file: 112 result_file.write(results + "\n") 113 results_array = [] 114 return 115 116 117def get_nvme_devices_count(): 118 output = check_output('lspci | grep -i Non | wc -l', shell=True) 119 return int(output) 120 121 122def get_nvme_devices_bdf(): 123 output = check_output('lspci | grep -i Non | awk \'{print $1}\'', shell=True).decode("utf-8") 124 output = output.split() 125 return output 126 127 128def add_filename_to_conf(conf_file_name, bdf): 129 filestring = "filename=trtype=PCIe traddr=0000." + bdf.replace(":", ".") + " ns=1" 130 with open(conf_file_name, "a") as conf_file: 131 conf_file.write(filestring + "\n") 132 133 134if len(sys.argv) != 4: 135 print("usage: " % sys.argv[0] % " path_to_fio_conf path_to_ioengine num_ssds") 136 sys.exit() 137 138num_ssds = int(sys.argv[3]) 139if num_ssds > get_nvme_devices_count(): 140 print("System does not have {} NVMe SSDs.".format(num_ssds)) 141 sys.exit() 142 143host_name = os.uname()[1] 144result_file_name = host_name + "_" + sys.argv[3] + "ssds_perf_output.csv" 145 146bdf = get_nvme_devices_bdf() 147config_file_for_test = sys.argv[1] + "_" + sys.argv[3] + "ssds" 148copyfile(sys.argv[1], config_file_for_test) 149 150# Add the number of threads to the fio config file 151with open(config_file_for_test, "a") as conf_file: 152 conf_file.write("numjobs=" + str(1) + "\n") 153 154# Add the NVMe bdf to the fio config file 155for i in range(0, num_ssds): 156 add_filename_to_conf(config_file_for_test, bdf[i]) 157 158# Set up for output 159columns = "IO_Size,Q_Depth,Workload_Mix,Workload_Type,Core_Mask,Run_Time,Run,Read_IOPS,Read_bw(KiB/s), \ 160 Read_Avg_lat(us),Read_Min_Lat(us),Read_Max_Lat(us),Write_IOPS,Write_bw(KiB/s),Write_Avg_lat(us), \ 161 Write_Min_Lat(us),Write_Max_Lat(us)" 162 163with open(result_file_name, "w+") as result_file: 164 result_file.write(columns + "\n") 165 166for i, (s, q, m, w, c, t) in enumerate(itertools.product(io_size, q_depth, mix, workload_type, core_mask, run_time)): 167 run_fio(s, q, m, c, i, w, t) 168 parse_results(s, q, m, c, i, w, t) 169 170result_file.close() 171