xref: /spdk/test/iscsi_tgt/calsoft/calsoft.py (revision c8a36b04cb1200512bc8b6875ecb4eebe756f9e3) 
1#!/usr/bin/env python3
2#  SPDX-License-Identifier: BSD-3-Clause
3#  Copyright (C) 2016 Intel Corporation
4#  All rights reserved.
5#
6
7import os
8import time
9import sys
10import subprocess
11import threading
12import json
13
14CALSOFT_BIN_PATH = "/usr/local/calsoft/iscsi-pcts-v1.5/bin"
15
16'''
1711/26/2015 disable tc_login_11_2 and tc_login_11_4
18RFC 7143 6.3
19Neither the initiator nor the target should attempt to declare or
20negotiate a parameter more than once during login, except for
21responses to specific keys that explicitly allow repeated key
22declarations (e.g., TargetAddress)
23
24The spec didn't make it clear what other keys could be re-declare
25Discussed this with UNH and get the conclusion that TargetName/
26TargetAddress/MaxRecvDataSegmentLength could be re-declare.
27'''
28'''
2912/1/2015 add tc_login_2_2 to known_failed_cases
30RFC 7143 6.1
31A standard-label MUST begin with a capital letter and must not exceed
3263 characters.
33key name: A standard-label
34'''
35'''
3606/10/2020 add tc_login_29_1 to known_failed_cases
37RFC 3720 12.19. DataSequenceInOrder
38Irrelevant when: SessionType=Discovery
39'''
40
41known_failed_cases = ['tc_ffp_15_2', 'tc_ffp_29_2', 'tc_ffp_29_3', 'tc_ffp_29_4',
42                      'tc_err_1_1', 'tc_err_1_2', 'tc_err_2_8',
43                      'tc_err_3_1', 'tc_err_3_2', 'tc_err_3_3',
44                      'tc_err_3_4', 'tc_err_5_1', 'tc_login_3_1',
45                      'tc_login_11_2', 'tc_login_11_4', 'tc_login_2_2', 'tc_login_29_1']
46
47
48def run_case(case, result_list, log_dir_path):
49    try:
50        case_log = subprocess.check_output("{}/{}".format(CALSOFT_BIN_PATH, case), stderr=subprocess.STDOUT, shell=True).decode('utf-8')
51    except subprocess.CalledProcessError as e:
52        result_list.append({"Name": case, "Result": "FAIL"})
53        case_log = e.output.decode('utf-8')
54    else:
55        result_list.append({"Name": case, "Result": "PASS"})
56    with open(log_dir_path + case + '.txt', 'w') as f:
57        f.write(case_log)
58
59
60def main():
61    if not os.path.exists(CALSOFT_BIN_PATH):
62        print("The Calsoft test suite is not available on this machine.")
63        sys.exit(1)
64
65    output_dir = sys.argv[1]
66    if len(sys.argv) > 2:
67        output_file = sys.argv[2]
68    else:
69        output_file = "%s/calsoft.json" % (output_dir)
70
71    log_dir = "%s/calsoft/" % output_dir
72
73    all_cases = [x for x in os.listdir(CALSOFT_BIN_PATH) if x.startswith('tc')]
74    all_cases.sort()
75    nopin_cases = ['tc_err_2_4', 'tc_err_8_2', 'tc_ffp_7_6_1', 'tc_ffp_7_7_3', 'tc_ffp_7_7_2',
76                   'tc_ffp_7_7_5', 'tc_ffp_7_6_4', 'tc_ffp_7_6_3', 'tc_ffp_7_7_4', 'tc_ffp_7_6_2',
77                   'tc_ffp_7_7_1', 'tc_ffp_7_7']
78
79    case_result_list = []
80
81    result = {"Calsoft iSCSI tests": case_result_list}
82
83    if not os.path.exists(log_dir):
84        os.mkdir(log_dir)
85    for case in known_failed_cases:
86        print("Skipping %s. It is known to fail." % (case))
87        case_result_list.append({"Name": case, "Result": "SKIP"})
88
89    thread_objs = []
90
91    # The Calsoft tests all pull their InitiatorName from the its.conf file.  We set the
92    # AllowDuplicatedIsid flag in the SPDK JSON config, to allow these tests to run in
93    # parallel where needed, but we only run tests in parallel that make sense - in this case
94    # only the nopin-related tests which take longer to run because of various timeouts.
95    serial_cases = list(set(all_cases) - set(known_failed_cases) - set(nopin_cases))
96    parallel_cases = nopin_cases
97
98    for test_case in serial_cases:
99        thread_obj = threading.Thread(target=run_case, args=(test_case, case_result_list, log_dir, ))
100        thread_obj.start()
101        thread_obj.join(30)
102        if thread_obj.is_alive():
103            # Thread is still alive, meaning the join() timeout expired.
104            print("Thread timeout")
105            exit(1)
106
107    for test_case in parallel_cases:
108        thread_obj = threading.Thread(target=run_case, args=(test_case, case_result_list, log_dir, ))
109        thread_obj.start()
110        time.sleep(0.02)
111        thread_objs.append(thread_obj)
112
113    end_time = time.time() + 30
114    while time.time() < end_time:
115        for thread_obj in thread_objs:
116            if thread_obj.is_alive():
117                break
118        else:
119            break
120    else:
121        print("Thread timeout")
122        exit(1)
123    with open(output_file, 'w') as f:
124        json.dump(obj=result, fp=f, indent=2)
125
126    failed = 0
127    for x in case_result_list:
128        if x["Result"] == "FAIL":
129            print("Test case %s failed." % (x["Name"]))
130            failed = 1
131    exit(failed)
132
133
134if __name__ == '__main__':
135    main()
136