xref: /spdk/test/vhost/windows/windows_scsi_compliance.py (revision b30d57cdad6d2bc75cc1e4e2ebbcebcb0d98dcfa) 
1#!/usr/bin/env python3
2import os
3import sys
4import re
5import pprint
6import collections
7
8os.chdir(os.path.join(os.path.dirname(sys.argv[0]), "results"))
9
10scsi_logs = filter(lambda x: x.endswith(".log"), os.listdir("./"))
11scsi_1_pattern = re.compile(r"(ASSERTION\s[1-9][\d+]?\.\d+\s)(.+\s)([\w\W]+?)(Result:\s)(\w+)", re.I | re.M)
12scsi_2_pattern = re.compile(r"(?:Start:\s)(ASSERTION:\s)?(.+)(?:,.+=\s)([\w\W]+?)(End:\s)(\w+)(,.*)", re.I | re.M)
13fails = []
14warns = []
15
16expected_warns = [
17    "MODE_SELECT_6_MODE_SENSE_6_Checking_Parameters_Savable_PS_bit",
18    "MODE_SELECT_10_MODE_SENSE_10_Checking_Parameters_Savable_PS_bit",
19    "MODE_SELECT_10_Changing_WCE",
20    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_WCE_has_been_cleared",
21    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_Saved_Values_have_changed",
22    "MODE_SELECT_10_setting_WCE",
23    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_WCE_has_been_set",
24    "MODE_SELECT_10_Attempting_to_restore_original_values",
25    "MODE_SELECT_10_MODE_SENSE_10_Verifying_values_were_restored",
26    "ASSERTION_VERIFY_16_Support_Test",
27]
28
29expected_fails = [
30    "ASSERTION_READ_6_Read-With-Disk-Cache-Cleared_Test",
31    "ASSERTION_READ_10_Read-With-Disk-Cache-Cleared_Test",
32    "ASSERTION_READ_16_Read-With-Disk-Cache-Cleared_Test",
33    "ASSERTION_INQUIRY_Checking_Identification_Descriptors_in_VPD_page_0x83",
34    "ASSERTION_VERIFY_10_Support_Test",
35]
36
37results = {"1": collections.OrderedDict(),
38           "2": collections.OrderedDict()}
39
40for log in scsi_logs:
41    # Choose regex pattern depending on tests version
42    pattern = scsi_1_pattern if "WIN_SCSI_1" in log else scsi_2_pattern
43
44    # Read log file contents
45    try:
46        with open(log, 'r') as fh:
47            fh = open(log, 'r')
48            log_text = fh.read()
49            # Dir name for saving split result files of currently processed log file
50            d_name = log.split(".")[0]
51            try:
52                os.mkdir(d_name)
53            except OSError:
54                pass
55    except IOError as e:
56        print("ERROR: While opening log file: {log_file}".format(log_file=log))
57        exit(1)
58
59    # Parse log file contents
60    matches_found = re.findall(pattern, log_text)
61    if len(matches_found) < 1:
62        print("ERROR: No results found in file {log_file}!".format(log_file=log))
63        exit(1)
64
65    # Go through output for each test from log file; parse and save to dict
66    for m in matches_found:
67        test_name = re.sub(r"\s+", "_", (m[0] + m[1]).strip())
68        test_name = re.sub(r"[():]", "", test_name)
69        test_name = test_name[0:-1] if "." in test_name[-1] else test_name
70        tc_result = m[4].upper()
71
72        if "FAIL" in tc_result.upper():
73            fails.append([log, test_name, tc_result])
74        elif "WARN" in tc_result.upper():
75            warns.append([log, test_name, tc_result])
76
77        # Save output to separate file
78        with open(os.path.join("./", d_name, test_name), 'w') as fh:
79            for line in m:
80                fh.write(line)
81
82        # Also save in dictionary for later use in generating HTML results summary
83        ver = "1" if "WIN_SCSI_1" in log else "2"
84        try:
85            results[ver][test_name][d_name] = tc_result
86        except KeyError:
87            results[ver][test_name] = collections.OrderedDict()
88            results[ver][test_name][d_name] = tc_result
89
90
91# Generate HTML file with results table
92with open(os.path.join("./", "results.html"), 'a') as fh:
93    html = "<html>"
94    for suite_ver in results.keys():
95        html += """"<h2> WIN_SCSI_{ver} </h2>
96        <table bgcolor=\"#ffffff\" border=\"1px solid black;>\"""".format(ver=suite_ver)
97
98        # Print header
99        html += "<tr><th>Test name</th>"
100        disks_header = set()
101
102        for _ in results[suite_ver].keys():
103            for disk in results[suite_ver][_].keys():
104                disks_header.add(disk)
105
106        for disk in disks_header:
107            html += "<th>{disk}</th>".format(disk=disk)
108        html += "</tr>"
109
110        # Print results
111        for test in results[suite_ver].keys():
112            html += "<tr><td>{f_name}</td>".format(f_name=test)
113            for disk in disks_header:
114                try:
115                    result = results[suite_ver][test][disk]
116
117                    html += "<td"
118                    if "PASS" in result:
119                        html += " bgcolor=\"#99ff33\">"
120                    else:
121                        html += " bgcolor=\"#ff5050\">"
122
123                    html += "<a href={file}>{result}</a>".format(result=result, file=os.path.join("./", disk, test))
124                    html += "</td>"
125
126                except KeyError:
127                    html += "<td bgcolor=\"#ffff99\"></br></td>"
128            html += "</tr>"
129        html += "</table></br>"
130    html += "</html>"
131    fh.write(html)
132
133if warns:
134    not_expected_warns = [w for w in warns if w[1] not in expected_warns and "WIN_SCSI_2" in w[0]]
135    print("INFO: Windows SCSI compliance warnings:")
136    pprint.pprint(warns, width=150)
137
138if fails:
139    not_expected_fails = [f for f in fails if f[1] not in expected_fails and "WIN_SCSI_2" in f[0]]
140    print("INFO: Windows SCSI compliance fails:")
141    pprint.pprint(fails, width=150)
142
143if not_expected_warns or not_expected_fails:
144    print("Not expected fails / warnings:")
145    pprint.pprint(not_expected_warns, width=150)
146    pprint.pprint(not_expected_fails, width=150)
147    exit(1)
148