xref: /spdk/test/vhost/windows/windows_scsi_compliance.py (revision 60982c759db49b4f4579f16e3b24df0725ba4b94) 
1#!/usr/bin/env python3
2#  SPDX-License-Identifier: BSD-3-Clause
3#  Copyright (C) 2019 Intel Corporation
4#  All rights reserved.
5#
6
7import os
8import sys
9import re
10import pprint
11import collections
12
13os.chdir(os.path.join(os.path.dirname(sys.argv[0]), "results"))
14
15scsi_logs = filter(lambda x: x.endswith(".log"), os.listdir("./"))
16scsi_1_pattern = re.compile(r"(ASSERTION\s[1-9][\d+]?\.\d+\s)(.+\s)([\w\W]+?)(Result:\s)(\w+)", re.I | re.M)
17scsi_2_pattern = re.compile(r"(?:Start:\s)(ASSERTION:\s)?(.+)(?:,.+=\s)([\w\W]+?)(End:\s)(\w+)(,.*)", re.I | re.M)
18fails = []
19warns = []
20
21expected_warns = [
22    "MODE_SELECT_6_MODE_SENSE_6_Checking_Parameters_Savable_PS_bit",
23    "MODE_SELECT_10_MODE_SENSE_10_Checking_Parameters_Savable_PS_bit",
24    "MODE_SELECT_10_Changing_WCE",
25    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_WCE_has_been_cleared",
26    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_Saved_Values_have_changed",
27    "MODE_SELECT_10_setting_WCE",
28    "MODE_SELECT_10_MODE_SENSE_10_Checking_that_WCE_has_been_set",
29    "MODE_SELECT_10_Attempting_to_restore_original_values",
30    "MODE_SELECT_10_MODE_SENSE_10_Verifying_values_were_restored",
31    "ASSERTION_VERIFY_16_Support_Test",
32]
33
34expected_fails = [
35    "ASSERTION_READ_6_Read-With-Disk-Cache-Cleared_Test",
36    "ASSERTION_READ_10_Read-With-Disk-Cache-Cleared_Test",
37    "ASSERTION_READ_16_Read-With-Disk-Cache-Cleared_Test",
38    "ASSERTION_INQUIRY_Checking_Identification_Descriptors_in_VPD_page_0x83",
39    "ASSERTION_VERIFY_10_Support_Test",
40]
41
42results = {"1": collections.OrderedDict(),
43           "2": collections.OrderedDict()}
44
45for log in scsi_logs:
46    # Choose regex pattern depending on tests version
47    pattern = scsi_1_pattern if "WIN_SCSI_1" in log else scsi_2_pattern
48
49    # Read log file contents
50    try:
51        with open(log, 'r') as fh:
52            fh = open(log, 'r')
53            log_text = fh.read()
54            # Dir name for saving split result files of currently processed log file
55            d_name = log.split(".")[0]
56            try:
57                os.mkdir(d_name)
58            except OSError:
59                pass
60    except IOError as e:
61        print("ERROR: While opening log file: {log_file}".format(log_file=log))
62        exit(1)
63
64    # Parse log file contents
65    matches_found = re.findall(pattern, log_text)
66    if len(matches_found) < 1:
67        print("ERROR: No results found in file {log_file}!".format(log_file=log))
68        exit(1)
69
70    # Go through output for each test from log file; parse and save to dict
71    for m in matches_found:
72        test_name = re.sub(r"\s+", "_", (m[0] + m[1]).strip())
73        test_name = re.sub(r"[():]", "", test_name)
74        test_name = test_name[0:-1] if "." in test_name[-1] else test_name
75        tc_result = m[4].upper()
76
77        if "FAIL" in tc_result.upper():
78            fails.append([log, test_name, tc_result])
79        elif "WARN" in tc_result.upper():
80            warns.append([log, test_name, tc_result])
81
82        # Save output to separate file
83        with open(os.path.join("./", d_name, test_name), 'w') as fh:
84            for line in m:
85                fh.write(line)
86
87        # Also save in dictionary for later use in generating HTML results summary
88        ver = "1" if "WIN_SCSI_1" in log else "2"
89        try:
90            results[ver][test_name][d_name] = tc_result
91        except KeyError:
92            results[ver][test_name] = collections.OrderedDict()
93            results[ver][test_name][d_name] = tc_result
94
95
96# Generate HTML file with results table
97with open(os.path.join("./", "results.html"), 'a') as fh:
98    html = "<html>"
99    for suite_ver in results.keys():
100        html += """"<h2> WIN_SCSI_{ver} </h2>
101        <table bgcolor=\"#ffffff\" border=\"1px solid black;>\"""".format(ver=suite_ver)
102
103        # Print header
104        html += "<tr><th>Test name</th>"
105        disks_header = set()
106
107        for _ in results[suite_ver].keys():
108            for disk in results[suite_ver][_].keys():
109                disks_header.add(disk)
110
111        for disk in disks_header:
112            html += "<th>{disk}</th>".format(disk=disk)
113        html += "</tr>"
114
115        # Print results
116        for test in results[suite_ver].keys():
117            html += "<tr><td>{f_name}</td>".format(f_name=test)
118            for disk in disks_header:
119                try:
120                    result = results[suite_ver][test][disk]
121
122                    html += "<td"
123                    if "PASS" in result:
124                        html += " bgcolor=\"#99ff33\">"
125                    else:
126                        html += " bgcolor=\"#ff5050\">"
127
128                    html += "<a href={file}>{result}</a>".format(result=result, file=os.path.join("./", disk, test))
129                    html += "</td>"
130
131                except KeyError:
132                    html += "<td bgcolor=\"#ffff99\"></br></td>"
133            html += "</tr>"
134        html += "</table></br>"
135    html += "</html>"
136    fh.write(html)
137
138if warns:
139    not_expected_warns = [w for w in warns if w[1] not in expected_warns and "WIN_SCSI_2" in w[0]]
140    print("INFO: Windows SCSI compliance warnings:")
141    pprint.pprint(warns, width=150)
142
143if fails:
144    not_expected_fails = [f for f in fails if f[1] not in expected_fails and "WIN_SCSI_2" in f[0]]
145    print("INFO: Windows SCSI compliance fails:")
146    pprint.pprint(fails, width=150)
147
148if not_expected_warns or not_expected_fails:
149    print("Not expected fails / warnings:")
150    pprint.pprint(not_expected_warns, width=150)
151    pprint.pprint(not_expected_fails, width=150)
152    exit(1)
153