xref: /spdk/scripts/perf/nvme/run_fio_test.py (revision f86f10757912918b8ba7b4b3bfdab1cd4c2d180c) 
1#!/usr/bin/env python
2
3# This script runs fio benchmark test on the local nvme device using the SPDK NVMe driver.
4# Prework: Run script/setup.sh to bind SSDs to SPDK driver.
5# Prework: Change any fio configurations in the template fio config file fio_test.conf
6# Output: A csv file <hostname>_<num ssds>_perf_output.csv
7
8import subprocess
9from subprocess import check_call, call, check_output, Popen, PIPE
10import random
11import os
12import sys
13import re
14import signal
15import getopt
16from datetime import datetime
17from itertools import *
18import csv
19import itertools
20from shutil import copyfile
21import json
22
23# Populate test parameters into these lists to run different workloads
24# The configuration below runs QD 1 & 128. To add QD 32 set q_depth=['1', '32', '128']
25q_depth = ['1', '128']
26# io_size specifies the size in bytes of the IO workload.
27# To add 64K IOs set io_size = ['4096', '65536']
28io_size = ['4096']
29workload_type = ['randrw']
30mix = ['100']
31core_mask = ['0x1']
32# run_time parameter specifies how long to run each test.
33# Set run_time = ['600'] to run the test for 10 minutes
34run_time = ['60']
35# iter_num parameter is used to run the test multiple times.
36# set iter_num = ['1', '2', '3'] to repeat each test 3 times
37iter_num = ['1']
38
39def run_fio(io_size_bytes, qd, rw_mix, cpu_mask, run_num, workload, run_time_sec):
40    print "Running Test: IO Size=", io_size_bytes, " QD=", qd, " Mix=", rw_mix, "CPU Mask=", cpu_mask
41    string = "s_" + str(io_size_bytes) + "_q_" + str(qd) + "_m_" + str(rw_mix) + "_c_" + str(cpu_mask) + "_run_" + str(run_num)
42
43    # Call fio
44    path_to_fio_conf = config_file_for_test
45    path_to_ioengine = sys.argv[2]
46    command = "BLK_SIZE=" + str(io_size_bytes) + " RW=" + str(workload) + " MIX=" + str(rw_mix) \
47        + " IODEPTH=" + str(qd) + " RUNTIME=" + str(run_time_sec) + " IOENGINE=" + path_to_ioengine \
48        + " fio " + str(path_to_fio_conf) + " -output=" + string + " -output-format=json"
49    output = subprocess.check_output(command, shell=True)
50
51    print "Finished Test: IO Size=", io_size_bytes, " QD=", qd, " Mix=", rw_mix, " CPU Mask=", cpu_mask
52    return
53
54def parse_results(io_size_bytes, qd, rw_mix, cpu_mask, run_num, workload, run_time_sec):
55    results_array = []
56
57    # If json file has results for multiple fio jobs pick the results from the right job
58    job_pos = 0
59
60    # generate the next result line that will be added to the output csv file
61    results = str(io_size_bytes) + "," + str(qd) + "," + str(rw_mix) + "," \
62        + str(workload) + "," + str(cpu_mask) + "," + str(run_time_sec) + "," + str(run_num)
63
64    # Read the results of this run from the test result file
65    string = "s_" + str(io_size_bytes) + "_q_" + str(qd) + "_m_" + str(rw_mix) + "_c_" + str(cpu_mask) + "_run_" + str(run_num)
66    with open(string) as json_file:
67        data = json.load(json_file)
68        job_name = data['jobs'][job_pos]['jobname']
69        # print "FIO job name: ", job_name
70        if 'lat_ns' in data['jobs'][job_pos]['read']:
71            lat = 'lat_ns'
72            lat_units = 'ns'
73        else:
74            lat = 'lat'
75            lat_units = 'us'
76        read_iops = float(data['jobs'][job_pos]['read']['iops'])
77        read_bw = float(data['jobs'][job_pos]['read']['bw'])
78        read_avg_lat = float(data['jobs'][job_pos]['read'][lat]['mean'])
79        read_min_lat = float(data['jobs'][job_pos]['read'][lat]['min'])
80        read_max_lat = float(data['jobs'][job_pos]['read'][lat]['max'])
81        write_iops = float(data['jobs'][job_pos]['write']['iops'])
82        write_bw = float(data['jobs'][job_pos]['write']['bw'])
83        write_avg_lat = float(data['jobs'][job_pos]['write'][lat]['mean'])
84        write_min_lat = float(data['jobs'][job_pos]['write'][lat]['min'])
85        write_max_lat = float(data['jobs'][job_pos]['write'][lat]['max'])
86        print "%-10s" % "IO Size", "%-10s" % "QD", "%-10s" % "Mix", \
87            "%-10s" % "Workload Type", "%-10s" % "CPU Mask", \
88            "%-10s" % "Run Time", "%-10s" % "Run Num", \
89            "%-15s" % "Read IOps", \
90            "%-10s" % "Read MBps", "%-15s" % "Read Avg. Lat(" + lat_units + ")", \
91            "%-15s" % "Read Min. Lat(" + lat_units + ")", "%-15s" % "Read Max. Lat(" + lat_units + ")", \
92            "%-15s" % "Write IOps", \
93            "%-10s" % "Write MBps", "%-15s" % "Write Avg. Lat(" + lat_units + ")", \
94            "%-15s" % "Write Min. Lat(" + lat_units + ")", "%-15s" % "Write Max. Lat(" + lat_units + ")"
95        print "%-10s" % io_size_bytes, "%-10s" % qd, "%-10s" % rw_mix, \
96            "%-10s" % workload, "%-10s" % cpu_mask, "%-10s" % run_time_sec, \
97            "%-10s" % run_num, "%-15s" % read_iops, "%-10s" % read_bw, \
98            "%-15s" % read_avg_lat, "%-15s" % read_min_lat, "%-15s" % read_max_lat, \
99            "%-15s" % write_iops, "%-10s" % write_bw, "%-15s" % write_avg_lat, \
100            "%-15s" % write_min_lat, "%-15s" % write_max_lat
101        results = results + "," + str(read_iops) + "," + str(read_bw) + "," \
102            + str(read_avg_lat) + "," + str(read_min_lat) + "," + str(read_max_lat) \
103            + "," + str(write_iops) + "," + str(write_bw) + "," + str(write_avg_lat) \
104            + "," + str(write_min_lat) + "," + str(write_max_lat)
105        with open(result_file_name, "a") as result_file:
106            result_file.write(results + "\n")
107        results_array = []
108    return
109
110def get_nvme_devices_count():
111    output = check_output('lspci | grep -i Non | wc -l', shell=True)
112    return int(output)
113
114def get_nvme_devices_bdf():
115    output = check_output('lspci | grep -i Non | awk \'{print $1}\'', shell=True)
116    output = output.split()
117    return output
118
119def add_filename_to_conf(conf_file_name, bdf):
120    filestring = "filename=trtype=PCIe traddr=0000." + bdf.replace(":", ".") + " ns=1"
121    with open(conf_file_name, "a") as conf_file:
122        conf_file.write(filestring + "\n")
123
124if len(sys.argv) != 4:
125    print "usage: python ", sys.argv[0], " path_to_fio_conf path_to_ioengine num_ssds"
126    sys.exit()
127
128num_ssds = int(sys.argv[3])
129if num_ssds > get_nvme_devices_count():
130    print "System does not have ", num_ssds, " NVMe SSDs."
131    sys.exit()
132
133host_name = os.uname()[1]
134result_file_name = host_name + "_" + sys.argv[3] + "ssds_perf_output.csv"
135
136bdf = get_nvme_devices_bdf()
137config_file_for_test = sys.argv[1] + "_" + sys.argv[3] + "ssds"
138copyfile(sys.argv[1], config_file_for_test)
139
140# Add the number of threads to the fio config file
141with open(config_file_for_test, "a") as conf_file:
142    conf_file.write("numjobs=" + str(1) + "\n")
143
144# Add the NVMe bdf to the fio config file
145for i in range(0, num_ssds):
146    add_filename_to_conf(config_file_for_test, bdf[i])
147
148# Set up for output
149columns = "IO_Size,Q_Depth,Workload_Mix,Workload_Type,Core_Mask,Run_Time,Run,Read_IOPS,Read_bw(KiB/s), \
150    Read_Avg_lat(us),Read_Min_Lat(us),Read_Max_Lat(us),Write_IOPS,Write_bw(KiB/s),Write_Avg_lat(us), \
151    Write_Min_Lat(us),Write_Max_Lat(us)"
152
153with open(result_file_name, "w+") as result_file:
154    result_file.write(columns + "\n")
155
156for i, (s, q, m, w, c, t) in enumerate(itertools.product(io_size, q_depth, mix, workload_type, core_mask, run_time)):
157    run_fio(s, q, m, c, i, w, t)
158    parse_results(s, q, m, c, i, w, t)
159
160result_file.close()
161