xref: /spdk/scripts/rpc.py (revision 31e3adbada135b1bcabfd77cfa7abd0742eaa278)
1#!/usr/bin/env python3
2#  SPDX-License-Identifier: BSD-3-Clause
3#  Copyright (C) 2016 Intel Corporation
4#  All rights reserved.
5#  Copyright (c) 2022 Dell Inc, or its subsidiaries.
6#  Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
7#
8
9import logging
10import argparse
11import importlib
12import os
13import sys
14import shlex
15import json
16
17try:
18    from shlex import quote
19except ImportError:
20    from pipes import quote
21
22sys.path.append(os.path.dirname(__file__) + '/../python')
23
24import spdk.rpc as rpc  # noqa
25from spdk.rpc.client import print_dict, print_json, JSONRPCException  # noqa
26from spdk.rpc.helpers import deprecated_aliases  # noqa
27
28
29def print_array(a):
30    print(" ".join((quote(v) for v in a)))
31
32
33if __name__ == "__main__":
34    parser = argparse.ArgumentParser(
35        description='SPDK RPC command line interface', usage='%(prog)s [options]')
36    parser.add_argument('-s', dest='server_addr',
37                        help='RPC domain socket path or IP address', default='/var/tmp/spdk.sock')
38    parser.add_argument('-p', dest='port',
39                        help='RPC port number (if server_addr is IP address)',
40                        default=5260, type=int)
41    parser.add_argument('-t', dest='timeout',
42                        help='Timeout as a floating point number expressed in seconds waiting for response. Default: 60.0',
43                        default=None, type=float)
44    parser.add_argument('-r', dest='conn_retries',
45                        help='Retry connecting to the RPC server N times with 0.2s interval. Default: 0',
46                        default=0, type=int)
47    parser.add_argument('-v', dest='verbose', action='store_const', const="INFO",
48                        help='Set verbose mode to INFO', default="ERROR")
49    parser.add_argument('--verbose', dest='verbose', choices=['DEBUG', 'INFO', 'ERROR'],
50                        help="""Set verbose level. """)
51    parser.add_argument('--dry-run', dest='dry_run', action='store_true', help="Display request and exit")
52    parser.set_defaults(dry_run=False)
53    parser.add_argument('--go-client', dest='go_client', action='store_true', help="Use Go client")
54    parser.set_defaults(go_client=False)
55    parser.add_argument('--server', dest='is_server', action='store_true',
56                        help="Start listening on stdin, parse each line as a regular rpc.py execution and create \
57                                a separate connection for each command. Each command's output ends with either \
58                                **STATUS=0 if the command succeeded or **STATUS=1 if it failed. --server is meant \
59                                to be used in conjunction with bash coproc, where stdin and stdout are connected to \
60                                pipes and can be used as a faster way to send RPC commands. If enabled, rpc.py \
61                                must be executed without any other parameters.")
62    parser.set_defaults(is_server=False)
63    parser.add_argument('--plugin', dest='rpc_plugin', help='Module name of plugin with additional RPC commands')
64    subparsers = parser.add_subparsers(help='RPC methods', dest='called_rpc_name', metavar='')
65
66    def framework_start_init(args):
67        rpc.framework_start_init(args.client)
68
69    p = subparsers.add_parser('framework_start_init', help='Start initialization of subsystems')
70    p.set_defaults(func=framework_start_init)
71
72    def framework_wait_init(args):
73        rpc.framework_wait_init(args.client)
74
75    p = subparsers.add_parser('framework_wait_init', help='Block until subsystems have been initialized')
76    p.set_defaults(func=framework_wait_init)
77
78    def rpc_get_methods(args):
79        print_dict(rpc.rpc_get_methods(args.client,
80                                       current=args.current,
81                                       include_aliases=args.include_aliases))
82
83    p = subparsers.add_parser('rpc_get_methods', help='Get list of supported RPC methods')
84    p.add_argument('-c', '--current', help='Get list of RPC methods only callable in the current state.', action='store_true')
85    p.add_argument('-i', '--include-aliases', help='include RPC aliases', action='store_true')
86    p.set_defaults(func=rpc_get_methods)
87
88    def spdk_get_version(args):
89        print_json(rpc.spdk_get_version(args.client))
90
91    p = subparsers.add_parser('spdk_get_version', help='Get SPDK version')
92    p.set_defaults(func=spdk_get_version)
93
94    def save_config(args):
95        rpc.save_config(args.client,
96                        sys.stdout,
97                        indent=args.indent,
98                        subsystems=args.subsystems)
99
100    p = subparsers.add_parser('save_config', help="""Write current (live) configuration of SPDK subsystems and targets to stdout.
101    """)
102    p.add_argument('-i', '--indent', help="""Indent level. Value less than 0 mean compact mode. Default indent level is 2.
103    """, type=int, default=2)
104    p.add_argument('-s', '--subsystems', help="""Comma-separated list of subsystems (and their dependencies) to save""")
105    p.set_defaults(func=save_config)
106
107    def load_config(args):
108        rpc.load_config(args.client, args.json_conf,
109                        include_aliases=args.include_aliases)
110
111    p = subparsers.add_parser('load_config', help="""Configure SPDK subsystems and targets using JSON RPC.""")
112    p.add_argument('-i', '--include-aliases', help='include RPC aliases', action='store_true')
113    p.add_argument('-j', '--json-conf', help='Valid JSON configuration', default=sys.stdin)
114    p.set_defaults(func=load_config)
115
116    def save_subsystem_config(args):
117        rpc.save_subsystem_config(args.client,
118                                  sys.stdout,
119                                  indent=args.indent,
120                                  name=args.name)
121
122    p = subparsers.add_parser('save_subsystem_config', help="""Write current (live) configuration of SPDK subsystem to stdout.
123    """)
124    p.add_argument('-i', '--indent', help="""Indent level. Value less than 0 mean compact mode. Default indent level is 2.
125    """, type=int, default=2)
126    p.add_argument('-n', '--name', help='Name of subsystem', required=True)
127    p.set_defaults(func=save_subsystem_config)
128
129    def load_subsystem_config(args):
130        rpc.load_subsystem_config(args.client,
131                                  args.json_conf)
132
133    p = subparsers.add_parser('load_subsystem_config', help="""Configure SPDK subsystem using JSON RPC.""")
134    p.add_argument('-j', '--json-conf', help='Valid JSON configuration', default=sys.stdin)
135    p.set_defaults(func=load_subsystem_config)
136
137    # app
138    def spdk_kill_instance(args):
139        rpc.app.spdk_kill_instance(args.client,
140                                   sig_name=args.sig_name)
141
142    p = subparsers.add_parser('spdk_kill_instance', help='Send signal to instance')
143    p.add_argument('sig_name', help='signal will be sent to server.')
144    p.set_defaults(func=spdk_kill_instance)
145
146    def framework_monitor_context_switch(args):
147        enabled = None
148        if args.enable:
149            enabled = True
150        if args.disable:
151            enabled = False
152        print_dict(rpc.app.framework_monitor_context_switch(args.client,
153                                                            enabled=enabled))
154
155    p = subparsers.add_parser('framework_monitor_context_switch',
156                              help='Control whether the context switch monitor is enabled')
157    p.add_argument('-e', '--enable', action='store_true', help='Enable context switch monitoring')
158    p.add_argument('-d', '--disable', action='store_true', help='Disable context switch monitoring')
159    p.set_defaults(func=framework_monitor_context_switch)
160
161    def framework_get_reactors(args):
162        print_dict(rpc.app.framework_get_reactors(args.client))
163
164    p = subparsers.add_parser(
165        'framework_get_reactors', help='Display list of all reactors')
166    p.set_defaults(func=framework_get_reactors)
167
168    def framework_set_scheduler(args):
169        rpc.app.framework_set_scheduler(args.client,
170                                        name=args.name,
171                                        period=args.period,
172                                        load_limit=args.load_limit,
173                                        core_limit=args.core_limit,
174                                        core_busy=args.core_busy)
175
176    p = subparsers.add_parser(
177        'framework_set_scheduler', help='Select thread scheduler that will be activated and its period (experimental)')
178    p.add_argument('name', help="Name of a scheduler")
179    p.add_argument('-p', '--period', help="Scheduler period in microseconds", type=int)
180    p.add_argument('--load-limit', help="Scheduler load limit. Reserved for dynamic scheduler", type=int)
181    p.add_argument('--core-limit', help="Scheduler core limit. Reserved for dynamic scheduler", type=int)
182    p.add_argument('--core-busy', help="Scheduler core busy limit. Reserved for dynamic scheduler", type=int)
183    p.set_defaults(func=framework_set_scheduler)
184
185    def framework_get_scheduler(args):
186        print_dict(rpc.app.framework_get_scheduler(args.client))
187
188    p = subparsers.add_parser(
189        'framework_get_scheduler', help='Display currently set scheduler and its properties.')
190    p.set_defaults(func=framework_get_scheduler)
191
192    def framework_get_governor(args):
193        print_dict(rpc.app.framework_get_governor(args.client))
194
195    p = subparsers.add_parser(
196        'framework_get_governor', help='Display currently set governor and the available, set CPU frequencies.')
197    p.set_defaults(func=framework_get_governor)
198
199    def scheduler_set_options(args):
200        rpc.app.scheduler_set_options(args.client,
201                                      isolated_core_mask=args.isolated_core_mask,
202                                      scheduling_core=args.scheduling_core)
203    p = subparsers.add_parser('scheduler_set_options', help='Set scheduler options')
204    p.add_argument('-i', '--isolated-core-mask', help="Mask of CPU cores to isolate from scheduling change", type=str)
205    p.add_argument('-s', '--scheduling-core', help="Scheduler scheduling core. Idle threads will move to scheduling core."
206                   "Reserved for dynamic scheduler.", type=int)
207    p.set_defaults(func=scheduler_set_options)
208
209    def framework_disable_cpumask_locks(args):
210        rpc.framework_disable_cpumask_locks(args.client)
211
212    p = subparsers.add_parser('framework_disable_cpumask_locks',
213                              help='Disable CPU core lock files.')
214    p.set_defaults(func=framework_disable_cpumask_locks)
215
216    def framework_enable_cpumask_locks(args):
217        rpc.framework_enable_cpumask_locks(args.client)
218
219    p = subparsers.add_parser('framework_enable_cpumask_locks',
220                              help='Enable CPU core lock files.')
221    p.set_defaults(func=framework_enable_cpumask_locks)
222
223    # bdev
224    def bdev_set_options(args):
225        rpc.bdev.bdev_set_options(args.client,
226                                  bdev_io_pool_size=args.bdev_io_pool_size,
227                                  bdev_io_cache_size=args.bdev_io_cache_size,
228                                  bdev_auto_examine=args.bdev_auto_examine,
229                                  iobuf_small_cache_size=args.iobuf_small_cache_size,
230                                  iobuf_large_cache_size=args.iobuf_large_cache_size)
231
232    p = subparsers.add_parser('bdev_set_options',
233                              help="""Set options of bdev subsystem""")
234    p.add_argument('-p', '--bdev-io-pool-size', help='Number of bdev_io structures in shared buffer pool', type=int)
235    p.add_argument('-c', '--bdev-io-cache-size', help='Maximum number of bdev_io structures cached per thread', type=int)
236    group = p.add_mutually_exclusive_group()
237    group.add_argument('-e', '--enable-auto-examine', dest='bdev_auto_examine', help='Allow to auto examine', action='store_true')
238    group.add_argument('-d', '--disable-auto-examine', dest='bdev_auto_examine', help='Not allow to auto examine', action='store_false')
239    p.add_argument('--iobuf-small-cache-size', help='Size of the small iobuf per thread cache', type=int)
240    p.add_argument('--iobuf-large-cache-size', help='Size of the large iobuf per thread cache', type=int)
241    p.set_defaults(bdev_auto_examine=True)
242    p.set_defaults(func=bdev_set_options)
243
244    def bdev_examine(args):
245        rpc.bdev.bdev_examine(args.client,
246                              name=args.name)
247
248    p = subparsers.add_parser('bdev_examine',
249                              help="""examine a bdev if it exists, or will examine it after it is created""")
250    p.add_argument('-b', '--name', help='Name or alias of the bdev')
251    p.set_defaults(func=bdev_examine)
252
253    def bdev_wait_for_examine(args):
254        rpc.bdev.bdev_wait_for_examine(args.client)
255
256    p = subparsers.add_parser('bdev_wait_for_examine',
257                              help="""Report when all bdevs have been examined""")
258    p.set_defaults(func=bdev_wait_for_examine)
259
260    def bdev_compress_create(args):
261        print_json(rpc.bdev.bdev_compress_create(args.client,
262                                                 base_bdev_name=args.base_bdev_name,
263                                                 pm_path=args.pm_path,
264                                                 lb_size=args.lb_size))
265
266    p = subparsers.add_parser('bdev_compress_create', help='Add a compress vbdev')
267    p.add_argument('-b', '--base-bdev-name', help="Name of the base bdev", required=True)
268    p.add_argument('-p', '--pm-path', help="Path to persistent memory", required=True)
269    p.add_argument('-l', '--lb-size', help="Compressed vol logical block size (optional, if used must be 512 or 4096)", type=int)
270    p.set_defaults(func=bdev_compress_create)
271
272    def bdev_compress_delete(args):
273        rpc.bdev.bdev_compress_delete(args.client,
274                                      name=args.name)
275
276    p = subparsers.add_parser('bdev_compress_delete', help='Delete a compress disk')
277    p.add_argument('name', help='compress bdev name')
278    p.set_defaults(func=bdev_compress_delete)
279
280    def bdev_compress_get_orphans(args):
281        print_dict(rpc.bdev.bdev_compress_get_orphans(args.client,
282                                                      name=args.name))
283    p = subparsers.add_parser(
284        'bdev_compress_get_orphans', help='Display list of orphaned compress bdevs.')
285    p.add_argument('-b', '--name', help="Name of a comp bdev. Example: COMP_Nvme0n1")
286    p.set_defaults(func=bdev_compress_get_orphans)
287
288    def bdev_crypto_create(args):
289        print_json(rpc.bdev.bdev_crypto_create(args.client,
290                                               base_bdev_name=args.base_bdev_name,
291                                               name=args.name,
292                                               crypto_pmd=args.crypto_pmd,
293                                               key=args.key,
294                                               cipher=args.cipher,
295                                               key2=args.key2,
296                                               key_name=args.key_name))
297    p = subparsers.add_parser('bdev_crypto_create', help='Add a crypto vbdev')
298    p.add_argument('base_bdev_name', help="Name of the base bdev")
299    p.add_argument('name', help="Name of the crypto vbdev")
300    p.add_argument('-p', '--crypto-pmd', help="Name of the crypto device driver. Obsolete, see dpdk_cryptodev_set_driver")
301    p.add_argument('-k', '--key', help="Key. Obsolete, see accel_crypto_key_create")
302    p.add_argument('-c', '--cipher', help="cipher to use. Obsolete, see accel_crypto_key_create")
303    p.add_argument('-k2', '--key2', help="2nd key for cipher AES_XTS. Obsolete, see accel_crypto_key_create", default=None)
304    p.add_argument('-n', '--key-name', help="Key name to use, see accel_crypto_key_create")
305    p.set_defaults(func=bdev_crypto_create)
306
307    def bdev_crypto_delete(args):
308        rpc.bdev.bdev_crypto_delete(args.client,
309                                    name=args.name)
310
311    p = subparsers.add_parser('bdev_crypto_delete', help='Delete a crypto disk')
312    p.add_argument('name', help='crypto bdev name')
313    p.set_defaults(func=bdev_crypto_delete)
314
315    def bdev_ocf_create(args):
316        print_json(rpc.bdev.bdev_ocf_create(args.client,
317                                            name=args.name,
318                                            mode=args.mode,
319                                            cache_line_size=args.cache_line_size,
320                                            cache_bdev_name=args.cache_bdev_name,
321                                            core_bdev_name=args.core_bdev_name))
322    p = subparsers.add_parser('bdev_ocf_create', help='Add an OCF block device')
323    p.add_argument('name', help='Name of resulting OCF bdev')
324    p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo'])
325    p.add_argument(
326        '--cache-line-size',
327        help='OCF cache line size. The unit is KiB',
328        type=int,
329        choices=[4, 8, 16, 32, 64]
330    )
331    p.add_argument('cache_bdev_name', help='Name of underlying cache bdev')
332    p.add_argument('core_bdev_name', help='Name of underlying core bdev')
333    p.set_defaults(func=bdev_ocf_create)
334
335    def bdev_ocf_delete(args):
336        rpc.bdev.bdev_ocf_delete(args.client,
337                                 name=args.name)
338
339    p = subparsers.add_parser('bdev_ocf_delete', help='Delete an OCF block device')
340    p.add_argument('name', help='Name of OCF bdev')
341    p.set_defaults(func=bdev_ocf_delete)
342
343    def bdev_ocf_get_stats(args):
344        print_dict(rpc.bdev.bdev_ocf_get_stats(args.client,
345                                               name=args.name))
346    p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device')
347    p.add_argument('name', help='Name of OCF bdev')
348    p.set_defaults(func=bdev_ocf_get_stats)
349
350    def bdev_ocf_reset_stats(args):
351        print_dict(rpc.bdev.bdev_ocf_reset_stats(args.client,
352                                                 name=args.name))
353    p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of chosen OCF block device')
354    p.add_argument('name', help='Name of OCF bdev')
355    p.set_defaults(func=bdev_ocf_reset_stats)
356
357    def bdev_ocf_get_bdevs(args):
358        print_dict(rpc.bdev.bdev_ocf_get_bdevs(args.client,
359                                               name=args.name))
360    p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get list of OCF devices including unregistered ones')
361    p.add_argument('name', nargs='?', help='name of OCF vbdev or name of cache device or name of core device (optional)')
362    p.set_defaults(func=bdev_ocf_get_bdevs)
363
364    def bdev_ocf_set_cache_mode(args):
365        print_json(rpc.bdev.bdev_ocf_set_cache_mode(args.client,
366                                                    name=args.name,
367                                                    mode=args.mode))
368    p = subparsers.add_parser('bdev_ocf_set_cache_mode',
369                              help='Set cache mode of OCF block device')
370    p.add_argument('name', help='Name of OCF bdev')
371    p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo'])
372    p.set_defaults(func=bdev_ocf_set_cache_mode)
373
374    def bdev_ocf_set_seqcutoff(args):
375        rpc.bdev.bdev_ocf_set_seqcutoff(args.client,
376                                        name=args.name,
377                                        policy=args.policy,
378                                        threshold=args.threshold,
379                                        promotion_count=args.promotion_count)
380    p = subparsers.add_parser('bdev_ocf_set_seqcutoff',
381                              help='Set sequential cutoff parameters on all cores for the given OCF cache device')
382    p.add_argument('name', help='Name of OCF cache bdev')
383    p.add_argument('-t', '--threshold', type=int,
384                   help='Activation threshold [KiB]')
385    p.add_argument('-c', '--promotion-count', type=int,
386                   help='Promotion request count')
387    p.add_argument('-p', '--policy', choices=['always', 'full', 'never'], required=True,
388                   help='Sequential cutoff policy')
389    p.set_defaults(func=bdev_ocf_set_seqcutoff)
390
391    def bdev_ocf_flush_start(args):
392        rpc.bdev.bdev_ocf_flush_start(args.client, name=args.name)
393    p = subparsers.add_parser('bdev_ocf_flush_start',
394                              help='Start flushing OCF cache device')
395    p.add_argument('name', help='Name of OCF bdev')
396    p.set_defaults(func=bdev_ocf_flush_start)
397
398    def bdev_ocf_flush_status(args):
399        print_json(rpc.bdev.bdev_ocf_flush_status(args.client, name=args.name))
400    p = subparsers.add_parser('bdev_ocf_flush_status',
401                              help='Get flush status of OCF cache device')
402    p.add_argument('name', help='Name of OCF bdev')
403    p.set_defaults(func=bdev_ocf_flush_status)
404
405    def bdev_malloc_create(args):
406        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
407        print_json(rpc.bdev.bdev_malloc_create(args.client,
408                                               num_blocks=int(num_blocks),
409                                               block_size=args.block_size,
410                                               physical_block_size=args.physical_block_size,
411                                               name=args.name,
412                                               uuid=args.uuid,
413                                               optimal_io_boundary=args.optimal_io_boundary,
414                                               md_size=args.md_size,
415                                               md_interleave=args.md_interleave,
416                                               dif_type=args.dif_type,
417                                               dif_is_head_of_md=args.dif_is_head_of_md,
418                                               dif_pi_format=args.dif_pi_format))
419    p = subparsers.add_parser('bdev_malloc_create', help='Create a bdev with malloc backend')
420    p.add_argument('-b', '--name', help="Name of the bdev")
421    p.add_argument('-u', '--uuid', help="UUID of the bdev")
422    p.add_argument(
423        'total_size', help='Size of malloc bdev in MB (float > 0)', type=float)
424    p.add_argument('block_size', help='Data block size for this bdev', type=int)
425    p.add_argument('-p', '--physical-block-size', help='Physical block size for this bdev.', type=int)
426    p.add_argument('-o', '--optimal-io-boundary', help="""Split on optimal IO boundary, in number of
427    blocks, default 0 (disabled)""", type=int)
428    p.add_argument('-m', '--md-size', type=int,
429                   help='Metadata size for this bdev (0, 8, 16, 32, 64, or 128). Default is 0.')
430    p.add_argument('-i', '--md-interleave', action='store_true',
431                   help='Metadata location, interleaved if set, and separated if omitted.')
432    p.add_argument('-t', '--dif-type', type=int, choices=[0, 1, 2, 3],
433                   help='Protection information type. Parameter --md-size needs'
434                        'to be set along --dif-type. Default=0 - no protection.')
435    p.add_argument('-d', '--dif-is-head-of-md', action='store_true',
436                   help='Protection information is in the first 8 bytes of metadata. Default=false.')
437    p.add_argument('-f', '--dif-pi-format', type=int, choices=[0, 1, 2],
438                   help='Protection infromation format. Parameter --dif-type needs to be set together.'
439                        '0=16b Guard PI, 1=32b Guard PI, 2=64b Guard PI. Default=0.')
440    p.set_defaults(func=bdev_malloc_create)
441
442    def bdev_malloc_delete(args):
443        rpc.bdev.bdev_malloc_delete(args.client,
444                                    name=args.name)
445
446    p = subparsers.add_parser('bdev_malloc_delete', help='Delete a malloc disk')
447    p.add_argument('name', help='malloc bdev name')
448    p.set_defaults(func=bdev_malloc_delete)
449
450    def bdev_null_create(args):
451        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
452        if args.dif_type and not args.md_size:
453            print("ERROR: --md-size must be > 0 when --dif-type is > 0")
454            exit(1)
455        print_json(rpc.bdev.bdev_null_create(args.client,
456                                             num_blocks=num_blocks,
457                                             block_size=args.block_size,
458                                             physical_block_size=args.physical_block_size,
459                                             name=args.name,
460                                             uuid=args.uuid,
461                                             md_size=args.md_size,
462                                             dif_type=args.dif_type,
463                                             dif_is_head_of_md=args.dif_is_head_of_md,
464                                             dif_pi_format=args.dif_pi_format))
465
466    p = subparsers.add_parser('bdev_null_create', help='Add a bdev with null backend')
467    p.add_argument('name', help='Block device name')
468    p.add_argument('-u', '--uuid', help='UUID of the bdev')
469    p.add_argument('total_size', help='Size of null bdev in MB (int > 0). Includes only data blocks.', type=int)
470    p.add_argument('block_size', help='Data block size for this bdev.', type=int)
471    p.add_argument('-p', '--physical-block-size', help='Physical block size for this bdev.', type=int)
472    p.add_argument('-m', '--md-size', type=int,
473                   help='Metadata size for this bdev. Default=0.')
474    p.add_argument('-t', '--dif-type', type=int, choices=[0, 1, 2, 3],
475                   help='Protection information type. Parameter --md-size needs'
476                        'to be set along --dif-type. Default=0 - no protection.')
477    p.add_argument('-d', '--dif-is-head-of-md', action='store_true',
478                   help='Protection information is in the first 8 bytes of metadata. Default=false.')
479    p.add_argument('-f', '--dif-pi-format', type=int, choices=[0, 1, 2],
480                   help='Protection infromation format. Parameter --dif-type needs to be set together.'
481                        '0=16b Guard PI, 1=32b Guard PI, 2=64b Guard PI. Default=0.')
482    p.set_defaults(func=bdev_null_create)
483
484    def bdev_null_delete(args):
485        rpc.bdev.bdev_null_delete(args.client,
486                                  name=args.name)
487
488    p = subparsers.add_parser('bdev_null_delete', help='Delete a null bdev')
489    p.add_argument('name', help='null bdev name')
490    p.set_defaults(func=bdev_null_delete)
491
492    def bdev_null_resize(args):
493        print_json(rpc.bdev.bdev_null_resize(args.client,
494                                             name=args.name,
495                                             new_size=int(args.new_size)))
496
497    p = subparsers.add_parser('bdev_null_resize',
498                              help='Resize a null bdev')
499    p.add_argument('name', help='null bdev name')
500    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
501    p.set_defaults(func=bdev_null_resize)
502
503    def bdev_aio_create(args):
504        print_json(rpc.bdev.bdev_aio_create(args.client,
505                                            filename=args.filename,
506                                            name=args.name,
507                                            block_size=args.block_size,
508                                            readonly=args.readonly,
509                                            fallocate=args.fallocate))
510
511    p = subparsers.add_parser('bdev_aio_create', help='Add a bdev with aio backend')
512    p.add_argument('filename', help='Path to device or file (ex: /dev/sda)')
513    p.add_argument('name', help='Block device name')
514    p.add_argument('block_size', help='Block size for this bdev', type=int, nargs='?')
515    p.add_argument("-r", "--readonly", action='store_true', help='Set this bdev as read-only')
516    p.add_argument("--fallocate", action='store_true', help='Support unmap/writezeros by fallocate')
517    p.set_defaults(func=bdev_aio_create)
518
519    def bdev_aio_rescan(args):
520        print_json(rpc.bdev.bdev_aio_rescan(args.client,
521                                            name=args.name))
522
523    p = subparsers.add_parser('bdev_aio_rescan', help='Rescan a bdev size with aio backend')
524    p.add_argument('name', help='Block device name')
525    p.set_defaults(func=bdev_aio_rescan)
526
527    def bdev_aio_delete(args):
528        rpc.bdev.bdev_aio_delete(args.client,
529                                 name=args.name)
530
531    p = subparsers.add_parser('bdev_aio_delete', help='Delete an aio disk')
532    p.add_argument('name', help='aio bdev name')
533    p.set_defaults(func=bdev_aio_delete)
534
535    def bdev_uring_create(args):
536        print_json(rpc.bdev.bdev_uring_create(args.client,
537                                              filename=args.filename,
538                                              name=args.name,
539                                              block_size=args.block_size,
540                                              uuid=args.uuid))
541
542    p = subparsers.add_parser('bdev_uring_create', help='Create a bdev with io_uring backend')
543    p.add_argument('filename', help='Path to device or file (ex: /dev/nvme0n1)')
544    p.add_argument('name', help='bdev name')
545    p.add_argument('block_size', help='Block size for this bdev', type=int, nargs='?')
546    p.add_argument('-u', '--uuid', help="UUID of the bdev")
547    p.set_defaults(func=bdev_uring_create)
548
549    def bdev_uring_rescan(args):
550        print_json(rpc.bdev.bdev_uring_rescan(args.client,
551                                              name=args.name))
552
553    p = subparsers.add_parser('bdev_uring_rescan', help='Rescan a bdev size with uring backend')
554    p.add_argument('name', help='Block device name')
555    p.set_defaults(func=bdev_uring_rescan)
556
557    def bdev_uring_delete(args):
558        rpc.bdev.bdev_uring_delete(args.client,
559                                   name=args.name)
560
561    p = subparsers.add_parser('bdev_uring_delete', help='Delete a uring bdev')
562    p.add_argument('name', help='uring bdev name')
563    p.set_defaults(func=bdev_uring_delete)
564
565    def bdev_xnvme_create(args):
566        print_json(rpc.bdev.bdev_xnvme_create(args.client,
567                                              filename=args.filename,
568                                              name=args.name,
569                                              io_mechanism=args.io_mechanism))
570
571    p = subparsers.add_parser('bdev_xnvme_create', help='Create a bdev with xNVMe backend')
572    p.add_argument('filename', help='Path to device or file (ex: /dev/nvme0n1)')
573    p.add_argument('name', help='name of xNVMe bdev to create')
574    p.add_argument('io_mechanism', help='IO mechanism to use (ex: libaio, io_uring, io_uring_cmd, etc.)')
575    p.add_argument('conserve_cpu', action='store_true', help='Whether or not to conserve CPU when polling')
576    p.set_defaults(func=bdev_xnvme_create)
577
578    def bdev_xnvme_delete(args):
579        rpc.bdev.bdev_xnvme_delete(args.client,
580                                   name=args.name)
581
582    p = subparsers.add_parser('bdev_xnvme_delete', help='Delete a xNVMe bdev')
583    p.add_argument('name', help='xNVMe bdev name')
584    p.set_defaults(func=bdev_xnvme_delete)
585
586    def bdev_nvme_set_options(args):
587        rpc.bdev.bdev_nvme_set_options(args.client,
588                                       action_on_timeout=args.action_on_timeout,
589                                       timeout_us=args.timeout_us,
590                                       timeout_admin_us=args.timeout_admin_us,
591                                       keep_alive_timeout_ms=args.keep_alive_timeout_ms,
592                                       arbitration_burst=args.arbitration_burst,
593                                       low_priority_weight=args.low_priority_weight,
594                                       medium_priority_weight=args.medium_priority_weight,
595                                       high_priority_weight=args.high_priority_weight,
596                                       nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us,
597                                       nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us,
598                                       io_queue_requests=args.io_queue_requests,
599                                       delay_cmd_submit=args.delay_cmd_submit,
600                                       transport_retry_count=args.transport_retry_count,
601                                       bdev_retry_count=args.bdev_retry_count,
602                                       transport_ack_timeout=args.transport_ack_timeout,
603                                       ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
604                                       reconnect_delay_sec=args.reconnect_delay_sec,
605                                       fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec,
606                                       disable_auto_failback=args.disable_auto_failback,
607                                       generate_uuids=args.generate_uuids,
608                                       transport_tos=args.transport_tos,
609                                       nvme_error_stat=args.nvme_error_stat,
610                                       rdma_srq_size=args.rdma_srq_size,
611                                       io_path_stat=args.io_path_stat,
612                                       allow_accel_sequence=args.allow_accel_sequence,
613                                       rdma_max_cq_size=args.rdma_max_cq_size,
614                                       rdma_cm_event_timeout_ms=args.rdma_cm_event_timeout_ms,
615                                       dhchap_digests=args.dhchap_digests,
616                                       dhchap_dhgroups=args.dhchap_dhgroups)
617
618    p = subparsers.add_parser('bdev_nvme_set_options',
619                              help='Set options for the bdev nvme type. This is startup command.')
620    p.add_argument('-a', '--action-on-timeout',
621                   help="Action to take on command time out. Valid values are: none, reset, abort")
622    p.add_argument('-t', '--timeout-us',
623                   help="Timeout for each command, in microseconds. If 0, don't track timeouts.", type=int)
624    p.add_argument('--timeout-admin-us',
625                   help="Timeout for each admin command, in microseconds. If 0, treat same as io timeouts.", type=int)
626    p.add_argument('-k', '--keep-alive-timeout-ms',
627                   help="Keep alive timeout period in millisecond. If 0, disable keep-alive.", type=int)
628    p.add_argument('--arbitration-burst',
629                   help='the value is expressed as a power of two', type=int)
630    p.add_argument('--low-priority-weight',
631                   help='the maximum number of commands that the controller may launch at one time from a low priority queue', type=int)
632    p.add_argument('--medium-priority-weight',
633                   help='the maximum number of commands that the controller may launch at one time from a medium priority queue', type=int)
634    p.add_argument('--high-priority-weight',
635                   help='the maximum number of commands that the controller may launch at one time from a high priority queue', type=int)
636    p.add_argument('-p', '--nvme-adminq-poll-period-us',
637                   help='How often the admin queue is polled for asynchronous events', type=int)
638    p.add_argument('-i', '--nvme-ioq-poll-period-us',
639                   help='How often to poll I/O queues for completions', type=int)
640    p.add_argument('-s', '--io-queue-requests',
641                   help='The number of requests allocated for each NVMe I/O queue. Default: 512', type=int)
642    p.add_argument('-d', '--disable-delay-cmd-submit',
643                   help='Disable delaying NVMe command submission, i.e. no batching of multiple commands',
644                   action='store_false', dest='delay_cmd_submit')
645    p.add_argument('-c', '--transport-retry-count',
646                   help='the number of attempts per I/O in the transport layer when an I/O fails.', type=int)
647    p.add_argument('-r', '--bdev-retry-count',
648                   help='the number of attempts per I/O in the bdev layer when an I/O fails. -1 means infinite retries.', type=int)
649    p.add_argument('-e', '--transport-ack-timeout',
650                   help="""Time to wait ack until packet retransmission for RDMA or until closes connection for TCP.
651                   Range 0-31 where 0 is driver-specific default value.""", type=int)
652    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
653                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
654                   -1 means infinite reconnect retries. 0 means no reconnect retry.
655                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
656                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
657                   reconnect_delay_sec.
658                   This can be overridden by bdev_nvme_attach_controller.""",
659                   type=int)
660    p.add_argument('-o', '--reconnect-delay-sec',
661                   help="""Time to delay a reconnect retry.
662                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
663                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
664                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
665                   less than ctrlr_loss_timeout_sec.
666                   This can be overridden by bdev_nvme_attach_controller.""",
667                   type=int)
668    p.add_argument('-u', '--fast-io-fail-timeout-sec',
669                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
670                   0 means no such timeout.
671                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
672                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.
673                   This can be overridden by bdev_nvme_attach_controller.""",
674                   type=int)
675    p.add_argument('-f', '--disable-auto-failback',
676                   help="""Disable automatic failback. bdev_nvme_set_preferred_path can be used to do manual failback.
677                   By default, immediately failback to the preferred I/O path if it restored.""",
678                   action='store_true')
679    p.add_argument('--generate-uuids',
680                   help="""Enable generation of unique identifiers for NVMe bdevs only if they do
681                   not provide UUID themselves. These strings are based on device serial number and
682                   namespace ID and will always be the same for that device.""", action='store_true')
683    p.add_argument('--transport-tos',
684                   help="""IPv4 Type of Service value. Only applicable for RDMA transports.
685                   The default is 0 which means no TOS is applied.""", type=int)
686    p.add_argument('-m', '--nvme-error-stat', help="Enable collecting NVMe error counts.", action='store_true')
687    p.add_argument('-q', '--rdma-srq-size',
688                   help='Set the size of a shared rdma receive queue. Default: 0 (disabled)', type=int)
689    p.add_argument('--io-path-stat',
690                   help="""Enable collecting I/O path stat of each io path.""",
691                   action='store_true')
692    p.add_argument('--allow-accel-sequence',
693                   help='''Allow NVMe bdevs to advertise support for accel sequences if the
694                   controller also supports them.''', action='store_true')
695    p.add_argument('--rdma-max-cq-size',
696                   help='The maximum size of a rdma completion queue. Default: 0 (unlimited)', type=int)
697    p.add_argument('--rdma-cm-event-timeout-ms',
698                   help='Time to wait for RDMA CM event. Only applicable for RDMA transports.', type=int)
699    p.add_argument('--dhchap-digests', help='Comma-separated list of allowed DH-HMAC-CHAP digests',
700                   type=lambda d: d.split(','))
701    p.add_argument('--dhchap-dhgroups', help='Comma-separated list of allowed DH-HMAC-CHAP DH groups',
702                   type=lambda d: d.split(','))
703
704    p.set_defaults(func=bdev_nvme_set_options)
705
706    def bdev_nvme_set_hotplug(args):
707        rpc.bdev.bdev_nvme_set_hotplug(args.client, enable=args.enable, period_us=args.period_us)
708
709    p = subparsers.add_parser('bdev_nvme_set_hotplug', help='Set hotplug options for bdev nvme type.')
710    p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)")
711    p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug")
712    p.add_argument('-r', '--period-us',
713                   help='How often the hotplug is processed for insert and remove events', type=int)
714    p.set_defaults(func=bdev_nvme_set_hotplug)
715
716    def bdev_nvme_attach_controller(args):
717        print_array(rpc.bdev.bdev_nvme_attach_controller(args.client,
718                                                         name=args.name,
719                                                         trtype=args.trtype,
720                                                         traddr=args.traddr,
721                                                         adrfam=args.adrfam,
722                                                         trsvcid=args.trsvcid,
723                                                         priority=args.priority,
724                                                         subnqn=args.subnqn,
725                                                         hostnqn=args.hostnqn,
726                                                         hostaddr=args.hostaddr,
727                                                         hostsvcid=args.hostsvcid,
728                                                         prchk_reftag=args.prchk_reftag,
729                                                         prchk_guard=args.prchk_guard,
730                                                         hdgst=args.hdgst,
731                                                         ddgst=args.ddgst,
732                                                         fabrics_connect_timeout_us=args.fabrics_connect_timeout_us,
733                                                         multipath=args.multipath,
734                                                         num_io_queues=args.num_io_queues,
735                                                         ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
736                                                         reconnect_delay_sec=args.reconnect_delay_sec,
737                                                         fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec,
738                                                         psk=args.psk,
739                                                         max_bdevs=args.max_bdevs,
740                                                         dhchap_key=args.dhchap_key,
741                                                         dhchap_ctrlr_key=args.dhchap_ctrlr_key,
742                                                         allow_unrecognized_csi=args.allow_unrecognized_csi))
743
744    p = subparsers.add_parser('bdev_nvme_attach_controller', help='Add bdevs with nvme backend')
745    p.add_argument('-b', '--name', help="Name of the NVMe controller, prefix for each bdev name", required=True)
746    p.add_argument('-t', '--trtype',
747                   help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
748    p.add_argument('-a', '--traddr',
749                   help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
750    p.add_argument('-f', '--adrfam',
751                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
752    p.add_argument('-s', '--trsvcid',
753                   help='NVMe-oF target trsvcid: e.g., a port number')
754    p.add_argument('-p', '--priority',
755                   help='NVMe-oF connection priority: e.g., a priority number')
756    p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
757    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
758    p.add_argument('-i', '--hostaddr',
759                   help='NVMe-oF host address: e.g., an ip address')
760    p.add_argument('-c', '--hostsvcid',
761                   help='NVMe-oF host svcid: e.g., a port number')
762    p.add_argument('-r', '--prchk-reftag',
763                   help='Enable checking of PI reference tag for I/O processing.', action='store_true')
764    p.add_argument('-g', '--prchk-guard',
765                   help='Enable checking of PI guard for I/O processing.', action='store_true')
766    p.add_argument('-e', '--hdgst',
767                   help='Enable TCP header digest.', action='store_true')
768    p.add_argument('-d', '--ddgst',
769                   help='Enable TCP data digest.', action='store_true')
770    p.add_argument('--fabrics-timeout', type=int, help='Fabrics connect timeout in microseconds',
771                   dest="fabrics_connect_timeout_us")
772    p.add_argument('-x', '--multipath', help='Set multipath behavior (disable, failover, multipath)')
773    p.add_argument('--num-io-queues', type=int, help='Set the number of IO queues to request during initialization.')
774    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
775                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
776                   -1 means infinite reconnect retries. 0 means no reconnect retry.
777                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
778                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
779                   reconnect_delay_sec.""",
780                   type=int)
781    p.add_argument('-o', '--reconnect-delay-sec',
782                   help="""Time to delay a reconnect retry.
783                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
784                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
785                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
786                   less than ctrlr_loss_timeout_sec.""",
787                   type=int)
788    p.add_argument('-u', '--fast-io-fail-timeout-sec',
789                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
790                   0 means no such timeout.
791                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
792                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.""",
793                   type=int)
794    p.add_argument('-k', '--psk',
795                   help="""Set PSK and enable TCP SSL socket implementation.  The PSK can either be a
796                   name of a key attached to the keyring or a path to a file containing the key.  The
797                   latter method is deprecated.""")
798    p.add_argument('-m', '--max-bdevs', type=int,
799                   help='The size of the name array for newly created bdevs. Default is 128',)
800    p.add_argument('--dhchap-key', help='DH-HMAC-CHAP key name')
801    p.add_argument('--dhchap-ctrlr-key', help='DH-HMAC-CHAP controller key name')
802    p.add_argument('-U', '--allow-unrecognized-csi', help="""Allow attaching namespaces with unrecognized command set identifiers.
803                   These will only support NVMe passthrough.""", action='store_true')
804
805    p.set_defaults(func=bdev_nvme_attach_controller)
806
807    def bdev_nvme_get_controllers(args):
808        print_dict(rpc.nvme.bdev_nvme_get_controllers(args.client,
809                                                      name=args.name))
810
811    p = subparsers.add_parser(
812        'bdev_nvme_get_controllers', help='Display current NVMe controllers list or required NVMe controller')
813    p.add_argument('-n', '--name', help="Name of the NVMe controller. Example: Nvme0")
814    p.set_defaults(func=bdev_nvme_get_controllers)
815
816    def bdev_nvme_detach_controller(args):
817        rpc.bdev.bdev_nvme_detach_controller(args.client,
818                                             name=args.name,
819                                             trtype=args.trtype,
820                                             traddr=args.traddr,
821                                             adrfam=args.adrfam,
822                                             trsvcid=args.trsvcid,
823                                             subnqn=args.subnqn,
824                                             hostaddr=args.hostaddr,
825                                             hostsvcid=args.hostsvcid)
826
827    p = subparsers.add_parser('bdev_nvme_detach_controller',
828                              help='Detach an NVMe controller and delete any associated bdevs')
829    p.add_argument('name', help="Name of the controller")
830    p.add_argument('-t', '--trtype',
831                   help='NVMe-oF target trtype: e.g., rdma, pcie')
832    p.add_argument('-a', '--traddr',
833                   help='NVMe-oF target address: e.g., an ip address or BDF')
834    p.add_argument('-f', '--adrfam',
835                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
836    p.add_argument('-s', '--trsvcid',
837                   help='NVMe-oF target trsvcid: e.g., a port number')
838    p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
839    p.add_argument('-i', '--hostaddr',
840                   help='NVMe-oF host address: e.g., an ip address')
841    p.add_argument('-c', '--hostsvcid',
842                   help='NVMe-oF host svcid: e.g., a port number')
843    p.set_defaults(func=bdev_nvme_detach_controller)
844
845    def bdev_nvme_reset_controller(args):
846        rpc.bdev.bdev_nvme_reset_controller(args.client,
847                                            name=args.name,
848                                            cntlid=args.cntlid)
849
850    p = subparsers.add_parser('bdev_nvme_reset_controller',
851                              help='Reset an NVMe controller or all NVMe controllers in an NVMe bdev controller')
852    p.add_argument('name', help="Name of the NVMe controller")
853    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
854    p.set_defaults(func=bdev_nvme_reset_controller)
855
856    def bdev_nvme_enable_controller(args):
857        rpc.bdev.bdev_nvme_enable_controller(args.client,
858                                             name=args.name,
859                                             cntlid=args.cntlid)
860
861    p = subparsers.add_parser('bdev_nvme_enable_controller',
862                              help='Enable an NVMe controller or all NVMe controllers in an NVMe bdev controller')
863    p.add_argument('name', help="Name of the NVMe controller")
864    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
865    p.set_defaults(func=bdev_nvme_enable_controller)
866
867    def bdev_nvme_disable_controller(args):
868        rpc.bdev.bdev_nvme_disable_controller(args.client,
869                                              name=args.name,
870                                              cntlid=args.cntlid)
871
872    p = subparsers.add_parser('bdev_nvme_disable_controller',
873                              help='Disable an NVMe controller or all NVMe controllers in an NVMe bdev controller')
874    p.add_argument('name', help="Name of the NVMe controller")
875    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
876    p.set_defaults(func=bdev_nvme_disable_controller)
877
878    def bdev_nvme_start_discovery(args):
879        rpc.bdev.bdev_nvme_start_discovery(args.client,
880                                           name=args.name,
881                                           trtype=args.trtype,
882                                           traddr=args.traddr,
883                                           adrfam=args.adrfam,
884                                           trsvcid=args.trsvcid,
885                                           hostnqn=args.hostnqn,
886                                           wait_for_attach=args.wait_for_attach,
887                                           attach_timeout_ms=args.attach_timeout_ms,
888                                           ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
889                                           reconnect_delay_sec=args.reconnect_delay_sec,
890                                           fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec)
891
892    p = subparsers.add_parser('bdev_nvme_start_discovery', help='Start automatic discovery')
893    p.add_argument('-b', '--name', help="Name of the NVMe controller prefix for each bdev name", required=True)
894    p.add_argument('-t', '--trtype',
895                   help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
896    p.add_argument('-a', '--traddr',
897                   help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
898    p.add_argument('-f', '--adrfam',
899                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
900    p.add_argument('-s', '--trsvcid',
901                   help='NVMe-oF target trsvcid: e.g., a port number')
902    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
903    p.add_argument('-w', '--wait-for-attach', action='store_true',
904                   help='Do not complete RPC until all discovered NVM subsystems are attached')
905    p.add_argument('-T', '--attach-timeout-ms', type=int,
906                   help="""Time to wait until the discovery and all discovered NVM subsystems
907                        are attached (default: 0, meaning wait indefinitely).  Automatically
908                        selects the --wait-for-attach option.""")
909    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
910                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
911                   -1 means infinite reconnect retries. 0 means no reconnect retry.
912                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
913                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
914                   reconnect_delay_sec.""",
915                   type=int)
916    p.add_argument('-o', '--reconnect-delay-sec',
917                   help="""Time to delay a reconnect retry.
918                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
919                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
920                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
921                   less than ctrlr_loss_timeout_sec.""",
922                   type=int)
923    p.add_argument('-u', '--fast-io-fail-timeout-sec',
924                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
925                   0 means no such timeout.
926                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
927                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.""",
928                   type=int)
929    p.set_defaults(func=bdev_nvme_start_discovery)
930
931    def bdev_nvme_stop_discovery(args):
932        rpc.bdev.bdev_nvme_stop_discovery(args.client, name=args.name)
933
934    p = subparsers.add_parser('bdev_nvme_stop_discovery', help='Stop automatic discovery')
935    p.add_argument('-b', '--name', help="Name of the service to stop", required=True)
936    p.set_defaults(func=bdev_nvme_stop_discovery)
937
938    def bdev_nvme_get_discovery_info(args):
939        print_dict(rpc.bdev.bdev_nvme_get_discovery_info(args.client))
940
941    p = subparsers.add_parser('bdev_nvme_get_discovery_info', help='Get information about the automatic discovery')
942    p.set_defaults(func=bdev_nvme_get_discovery_info)
943
944    def bdev_nvme_get_io_paths(args):
945        print_dict(rpc.bdev.bdev_nvme_get_io_paths(args.client, name=args.name))
946
947    p = subparsers.add_parser('bdev_nvme_get_io_paths', help='Display active I/O paths')
948    p.add_argument('-n', '--name', help="Name of the NVMe bdev")
949    p.set_defaults(func=bdev_nvme_get_io_paths)
950
951    def bdev_nvme_set_preferred_path(args):
952        rpc.bdev.bdev_nvme_set_preferred_path(args.client,
953                                              name=args.name,
954                                              cntlid=args.cntlid)
955
956    p = subparsers.add_parser('bdev_nvme_set_preferred_path',
957                              help="""Set the preferred I/O path for an NVMe bdev when in multipath mode""")
958    p.add_argument('-b', '--name', help='Name of the NVMe bdev', required=True)
959    p.add_argument('-c', '--cntlid', help='NVMe-oF controller ID', type=int, required=True)
960    p.set_defaults(func=bdev_nvme_set_preferred_path)
961
962    def bdev_nvme_set_multipath_policy(args):
963        rpc.bdev.bdev_nvme_set_multipath_policy(args.client,
964                                                name=args.name,
965                                                policy=args.policy,
966                                                selector=args.selector,
967                                                rr_min_io=args.rr_min_io)
968
969    p = subparsers.add_parser('bdev_nvme_set_multipath_policy',
970                              help="""Set multipath policy of the NVMe bdev""")
971    p.add_argument('-b', '--name', help='Name of the NVMe bdev', required=True)
972    p.add_argument('-p', '--policy', help='Multipath policy (active_passive or active_active)', required=True)
973    p.add_argument('-s', '--selector', help='Multipath selector (round_robin, queue_depth)')
974    p.add_argument('-r', '--rr-min-io',
975                   help='Number of IO to route to a path before switching to another for round-robin',
976                   type=int)
977    p.set_defaults(func=bdev_nvme_set_multipath_policy)
978
979    def bdev_nvme_get_path_iostat(args):
980        print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client,
981                                                      name=args.name))
982
983    p = subparsers.add_parser('bdev_nvme_get_path_iostat',
984                              help="""Display current I/O statistics of all the IO paths of the blockdev. It can be
985                              called when io_path_stat is true.""")
986    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True)
987    p.set_defaults(func=bdev_nvme_get_path_iostat)
988
989    def bdev_nvme_cuse_register(args):
990        rpc.bdev.bdev_nvme_cuse_register(args.client,
991                                         name=args.name)
992
993    p = subparsers.add_parser('bdev_nvme_cuse_register',
994                              help='Register CUSE devices on NVMe controller')
995    p.add_argument('-n', '--name',
996                   help='Name of the NVMe controller. Example: Nvme0', required=True)
997    p.set_defaults(func=bdev_nvme_cuse_register)
998
999    def bdev_nvme_cuse_unregister(args):
1000        rpc.bdev.bdev_nvme_cuse_unregister(args.client,
1001                                           name=args.name)
1002
1003    p = subparsers.add_parser('bdev_nvme_cuse_unregister',
1004                              help='Unregister CUSE devices on NVMe controller')
1005    p.add_argument('-n', '--name',
1006                   help='Name of the NVMe controller. Example: Nvme0', required=True)
1007    p.set_defaults(func=bdev_nvme_cuse_unregister)
1008
1009    def bdev_zone_block_create(args):
1010        print_json(rpc.bdev.bdev_zone_block_create(args.client,
1011                                                   name=args.name,
1012                                                   base_bdev=args.base_bdev,
1013                                                   zone_capacity=args.zone_capacity,
1014                                                   optimal_open_zones=args.optimal_open_zones))
1015
1016    p = subparsers.add_parser('bdev_zone_block_create',
1017                              help='Create virtual zone namespace device with block device backend')
1018    p.add_argument('-b', '--name', help="Name of the zone device", required=True)
1019    p.add_argument('-n', '--base-bdev', help='Name of underlying, non-zoned bdev', required=True)
1020    p.add_argument('-z', '--zone-capacity', help='Surfaced zone capacity in blocks', type=int, required=True)
1021    p.add_argument('-o', '--optimal-open-zones', help='Number of zones required to reach optimal write speed', type=int, required=True)
1022    p.set_defaults(func=bdev_zone_block_create)
1023
1024    def bdev_zone_block_delete(args):
1025        rpc.bdev.bdev_zone_block_delete(args.client,
1026                                        name=args.name)
1027
1028    p = subparsers.add_parser('bdev_zone_block_delete', help='Delete a virtual zone namespace device')
1029    p.add_argument('name', help='Virtual zone bdev name')
1030    p.set_defaults(func=bdev_zone_block_delete)
1031
1032    def bdev_rbd_register_cluster(args):
1033        config_param = None
1034        if args.config_param:
1035            config_param = {}
1036            for entry in args.config_param:
1037                parts = entry.split('=', 1)
1038                if len(parts) != 2:
1039                    raise Exception('--config %s not in key=value form' % entry)
1040                config_param[parts[0]] = parts[1]
1041        print_json(rpc.bdev.bdev_rbd_register_cluster(args.client,
1042                                                      name=args.name,
1043                                                      user_id=args.user,
1044                                                      config_param=config_param,
1045                                                      config_file=args.config_file,
1046                                                      key_file=args.key_file,
1047                                                      core_mask=args.core_mask))
1048
1049    p = subparsers.add_parser('bdev_rbd_register_cluster',
1050                              help='Add a Rados cluster with ceph rbd backend')
1051    p.add_argument('name', help="Name of the Rados cluster only known to rbd bdev")
1052    p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)")
1053    p.add_argument('--config-param', action='append', metavar='key=value',
1054                   help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
1055    p.add_argument('--config-file', help="The file path of the Rados configuration file")
1056    p.add_argument('--key-file', help="The file path of the Rados keyring file")
1057    p.add_argument('--core-mask', help="Set core mask for librbd IO context threads")
1058    p.set_defaults(func=bdev_rbd_register_cluster)
1059
1060    def bdev_rbd_unregister_cluster(args):
1061        rpc.bdev.bdev_rbd_unregister_cluster(args.client, name=args.name)
1062
1063    p = subparsers.add_parser('bdev_rbd_unregister_cluster',
1064                              help='Unregister a Rados cluster object')
1065    p.add_argument('name', help='Name of the Rados Cluster only known to rbd bdev')
1066    p.set_defaults(func=bdev_rbd_unregister_cluster)
1067
1068    def bdev_rbd_get_clusters_info(args):
1069        print_json(rpc.bdev.bdev_rbd_get_clusters_info(args.client, name=args.name))
1070
1071    p = subparsers.add_parser('bdev_rbd_get_clusters_info',
1072                              help='Display registered Rados Cluster names and related info')
1073    p.add_argument('-b', '--name', help="Name of the registered Rados Cluster Name. Example: Cluster1")
1074    p.set_defaults(func=bdev_rbd_get_clusters_info)
1075
1076    def bdev_rbd_create(args):
1077        config = None
1078        if args.config:
1079            config = {}
1080            for entry in args.config:
1081                parts = entry.split('=', 1)
1082                if len(parts) != 2:
1083                    raise Exception('--config %s not in key=value form' % entry)
1084                config[parts[0]] = parts[1]
1085        print_json(rpc.bdev.bdev_rbd_create(args.client,
1086                                            name=args.name,
1087                                            user_id=args.user,
1088                                            config=config,
1089                                            pool_name=args.pool_name,
1090                                            rbd_name=args.rbd_name,
1091                                            block_size=args.block_size,
1092                                            cluster_name=args.cluster_name,
1093                                            uuid=args.uuid))
1094
1095    p = subparsers.add_parser('bdev_rbd_create', help='Add a bdev with ceph rbd backend')
1096    p.add_argument('-b', '--name', help="Name of the bdev")
1097    p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)")
1098    p.add_argument('--config', action='append', metavar='key=value',
1099                   help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
1100    p.add_argument('pool_name', help='rbd pool name')
1101    p.add_argument('rbd_name', help='rbd image name')
1102    p.add_argument('block_size', help='rbd block size', type=int)
1103    p.add_argument('-c', '--cluster-name', help="cluster name to identify the Rados cluster")
1104    p.add_argument('-u', '--uuid', help="UUID of the bdev")
1105    p.set_defaults(func=bdev_rbd_create)
1106
1107    def bdev_rbd_delete(args):
1108        rpc.bdev.bdev_rbd_delete(args.client,
1109                                 name=args.name)
1110
1111    p = subparsers.add_parser('bdev_rbd_delete', help='Delete a rbd bdev')
1112    p.add_argument('name', help='rbd bdev name')
1113    p.set_defaults(func=bdev_rbd_delete)
1114
1115    def bdev_rbd_resize(args):
1116        print_json(rpc.bdev.bdev_rbd_resize(args.client,
1117                                            name=args.name,
1118                                            new_size=int(args.new_size)))
1119
1120    p = subparsers.add_parser('bdev_rbd_resize',
1121                              help='Resize a rbd bdev')
1122    p.add_argument('name', help='rbd bdev name')
1123    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
1124    p.set_defaults(func=bdev_rbd_resize)
1125
1126    def bdev_delay_create(args):
1127        print_json(rpc.bdev.bdev_delay_create(args.client,
1128                                              base_bdev_name=args.base_bdev_name,
1129                                              name=args.name,
1130                                              uuid=args.uuid,
1131                                              avg_read_latency=args.avg_read_latency,
1132                                              p99_read_latency=args.nine_nine_read_latency,
1133                                              avg_write_latency=args.avg_write_latency,
1134                                              p99_write_latency=args.nine_nine_write_latency))
1135
1136    p = subparsers.add_parser('bdev_delay_create',
1137                              help='Add a delay bdev on existing bdev')
1138    p.add_argument('-b', '--base-bdev-name', help="Name of the existing bdev", required=True)
1139    p.add_argument('-d', '--name', help="Name of the delay bdev", required=True)
1140    p.add_argument('-u', '--uuid', help='UUID of the bdev (optional)')
1141    p.add_argument('-r', '--avg-read-latency',
1142                   help="Average latency to apply before completing read ops (in microseconds)", required=True, type=int)
1143    p.add_argument('-t', '--nine-nine-read-latency',
1144                   help="latency to apply to 1 in 100 read ops (in microseconds)", required=True, type=int)
1145    p.add_argument('-w', '--avg-write-latency',
1146                   help="Average latency to apply before completing write ops (in microseconds)", required=True, type=int)
1147    p.add_argument('-n', '--nine-nine-write-latency',
1148                   help="latency to apply to 1 in 100 write ops (in microseconds)", required=True, type=int)
1149    p.set_defaults(func=bdev_delay_create)
1150
1151    def bdev_delay_delete(args):
1152        rpc.bdev.bdev_delay_delete(args.client,
1153                                   name=args.name)
1154
1155    p = subparsers.add_parser('bdev_delay_delete', help='Delete a delay bdev')
1156    p.add_argument('name', help='delay bdev name')
1157    p.set_defaults(func=bdev_delay_delete)
1158
1159    def bdev_delay_update_latency(args):
1160        print_json(rpc.bdev.bdev_delay_update_latency(args.client,
1161                                                      delay_bdev_name=args.delay_bdev_name,
1162                                                      latency_type=args.latency_type,
1163                                                      latency_us=args.latency_us))
1164    p = subparsers.add_parser('bdev_delay_update_latency',
1165                              help='Update one of the latency values for a given delay bdev')
1166    p.add_argument('delay_bdev_name', help='The name of the given delay bdev')
1167    p.add_argument('latency_type', help='one of: avg_read, avg_write, p99_read, p99_write. No other values accepted.')
1168    p.add_argument('latency_us', help='new latency value in microseconds.', type=int)
1169    p.set_defaults(func=bdev_delay_update_latency)
1170
1171    def bdev_error_create(args):
1172        print_json(rpc.bdev.bdev_error_create(args.client,
1173                                              base_name=args.base_name,
1174                                              uuid=args.uuid))
1175
1176    p = subparsers.add_parser('bdev_error_create', help='Add bdev with error injection backend')
1177    p.add_argument('base_name', help='base bdev name')
1178    p.add_argument('--uuid', help='UUID for this bdev')
1179    p.set_defaults(func=bdev_error_create)
1180
1181    def bdev_error_delete(args):
1182        rpc.bdev.bdev_error_delete(args.client,
1183                                   name=args.name)
1184
1185    p = subparsers.add_parser('bdev_error_delete', help='Delete an error bdev')
1186    p.add_argument('name', help='error bdev name')
1187    p.set_defaults(func=bdev_error_delete)
1188
1189    def bdev_iscsi_set_options(args):
1190        rpc.bdev.bdev_iscsi_set_options(args.client,
1191                                        timeout_sec=args.timeout_sec)
1192
1193    p = subparsers.add_parser('bdev_iscsi_set_options', help='Set options for the bdev iscsi type.')
1194    p.add_argument('-t', '--timeout-sec', help="Timeout for command, in seconds, if 0, don't track timeout.", type=int)
1195    p.set_defaults(func=bdev_iscsi_set_options)
1196
1197    def bdev_iscsi_create(args):
1198        print_json(rpc.bdev.bdev_iscsi_create(args.client,
1199                                              name=args.name,
1200                                              url=args.url,
1201                                              initiator_iqn=args.initiator_iqn))
1202
1203    p = subparsers.add_parser('bdev_iscsi_create',
1204                              help='Add bdev with iSCSI initiator backend')
1205    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
1206    p.add_argument('-i', '--initiator-iqn', help="Initiator IQN", required=True)
1207    p.add_argument('--url', help="iSCSI Lun URL", required=True)
1208    p.set_defaults(func=bdev_iscsi_create)
1209
1210    def bdev_iscsi_delete(args):
1211        rpc.bdev.bdev_iscsi_delete(args.client,
1212                                   name=args.name)
1213
1214    p = subparsers.add_parser('bdev_iscsi_delete', help='Delete an iSCSI bdev')
1215    p.add_argument('name', help='iSCSI bdev name')
1216    p.set_defaults(func=bdev_iscsi_delete)
1217
1218    def bdev_passthru_create(args):
1219        print_json(rpc.bdev.bdev_passthru_create(args.client,
1220                                                 base_bdev_name=args.base_bdev_name,
1221                                                 name=args.name,
1222                                                 uuid=args.uuid))
1223
1224    p = subparsers.add_parser('bdev_passthru_create', help='Add a pass through bdev on existing bdev')
1225    p.add_argument('-b', '--base-bdev-name', help="Name of the existing bdev", required=True)
1226    p.add_argument('-p', '--name', help="Name of the pass through bdev", required=True)
1227    p.add_argument('-u', '--uuid', help="UUID of the bdev")
1228    p.set_defaults(func=bdev_passthru_create)
1229
1230    def bdev_passthru_delete(args):
1231        rpc.bdev.bdev_passthru_delete(args.client,
1232                                      name=args.name)
1233
1234    p = subparsers.add_parser('bdev_passthru_delete', help='Delete a pass through bdev')
1235    p.add_argument('name', help='pass through bdev name')
1236    p.set_defaults(func=bdev_passthru_delete)
1237
1238    def bdev_get_bdevs(args):
1239        print_dict(rpc.bdev.bdev_get_bdevs(args.client,
1240                                           name=args.name, timeout=args.timeout_ms))
1241
1242    p = subparsers.add_parser('bdev_get_bdevs',
1243                              help='Display current blockdev list or required blockdev')
1244    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1")
1245    p.add_argument('-t', '--timeout-ms', help="""Time in ms to wait for the bdev to appear (only used
1246    with the -b|--name option). The default timeout is 0, meaning the RPC returns immediately
1247    whether the bdev exists or not.""",
1248                   type=int)
1249    p.set_defaults(func=bdev_get_bdevs)
1250
1251    def bdev_get_iostat(args):
1252        print_dict(rpc.bdev.bdev_get_iostat(args.client,
1253                                            name=args.name,
1254                                            per_channel=args.per_channel))
1255
1256    p = subparsers.add_parser('bdev_get_iostat',
1257                              help='Display current I/O statistics of all the blockdevs or specified blockdev.')
1258    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1")
1259    p.add_argument('-c', '--per-channel', default=False, dest='per_channel', help='Display per channel IO stats for specified device',
1260                   action='store_true')
1261    p.set_defaults(func=bdev_get_iostat)
1262
1263    def bdev_reset_iostat(args):
1264        rpc.bdev.bdev_reset_iostat(args.client, name=args.name, mode=args.mode)
1265
1266    p = subparsers.add_parser('bdev_reset_iostat',
1267                              help='Reset I/O statistics of all the blockdevs or specified blockdev.')
1268    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1")
1269    p.add_argument('-m', '--mode', help="Mode to reset I/O statistics", choices=['all', 'maxmin'])
1270    p.set_defaults(func=bdev_reset_iostat)
1271
1272    def bdev_enable_histogram(args):
1273        rpc.bdev.bdev_enable_histogram(args.client, name=args.name, enable=args.enable, opc=args.opc)
1274
1275    p = subparsers.add_parser('bdev_enable_histogram',
1276                              help='Enable or disable histogram for specified bdev')
1277    p.add_argument('-e', '--enable', default=True, dest='enable', action='store_true', help='Enable histograms on specified device')
1278    p.add_argument('-d', '--disable', dest='enable', action='store_false', help='Disable histograms on specified device')
1279    p.add_argument('-o', '--opc', help='Enable histogram for specified io type. Defaults to all io types if not specified.'
1280                   ' Refer to bdev_get_bdevs RPC for the list of io types.')
1281    p.add_argument('name', help='bdev name')
1282    p.set_defaults(func=bdev_enable_histogram)
1283
1284    def bdev_get_histogram(args):
1285        print_dict(rpc.bdev.bdev_get_histogram(args.client, name=args.name))
1286
1287    p = subparsers.add_parser('bdev_get_histogram',
1288                              help='Get histogram for specified bdev')
1289    p.add_argument('name', help='bdev name')
1290    p.set_defaults(func=bdev_get_histogram)
1291
1292    def bdev_set_qd_sampling_period(args):
1293        rpc.bdev.bdev_set_qd_sampling_period(args.client,
1294                                             name=args.name,
1295                                             period=args.period)
1296
1297    p = subparsers.add_parser('bdev_set_qd_sampling_period',
1298                              help="Enable or disable tracking of a bdev's queue depth.")
1299    p.add_argument('name', help='Blockdev name. Example: Malloc0')
1300    p.add_argument('period', help='Period with which to poll the block device queue depth in microseconds.'
1301                   ' If set to 0, polling will be disabled.',
1302                   type=int)
1303    p.set_defaults(func=bdev_set_qd_sampling_period)
1304
1305    def bdev_set_qos_limit(args):
1306        rpc.bdev.bdev_set_qos_limit(args.client,
1307                                    name=args.name,
1308                                    rw_ios_per_sec=args.rw_ios_per_sec,
1309                                    rw_mbytes_per_sec=args.rw_mbytes_per_sec,
1310                                    r_mbytes_per_sec=args.r_mbytes_per_sec,
1311                                    w_mbytes_per_sec=args.w_mbytes_per_sec)
1312
1313    p = subparsers.add_parser('bdev_set_qos_limit',
1314                              help='Set QoS rate limit on a blockdev')
1315    p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0')
1316    p.add_argument('--rw-ios-per-sec',
1317                   help='R/W IOs per second limit (>=1000, example: 20000). 0 means unlimited.',
1318                   type=int)
1319    p.add_argument('--rw-mbytes-per-sec',
1320                   help="R/W megabytes per second limit (>=1, example: 100). 0 means unlimited.",
1321                   type=int)
1322    p.add_argument('--r-mbytes-per-sec',
1323                   help="Read megabytes per second limit (>=1, example: 100). 0 means unlimited.",
1324                   type=int)
1325    p.add_argument('--w-mbytes-per-sec',
1326                   help="Write megabytes per second limit (>=1, example: 100). 0 means unlimited.",
1327                   type=int)
1328    p.set_defaults(func=bdev_set_qos_limit)
1329
1330    def bdev_error_inject_error(args):
1331        rpc.bdev.bdev_error_inject_error(args.client,
1332                                         name=args.name,
1333                                         io_type=args.io_type,
1334                                         error_type=args.error_type,
1335                                         num=args.num,
1336                                         queue_depth=args.queue_depth,
1337                                         corrupt_offset=args.corrupt_offset,
1338                                         corrupt_value=args.corrupt_value)
1339
1340    p = subparsers.add_parser('bdev_error_inject_error', help='bdev inject error')
1341    p.add_argument('name', help="""the name of the error injection bdev""")
1342    p.add_argument('io_type', help="""io_type: 'clear' 'read' 'write' 'unmap' 'flush' 'all'""")
1343    p.add_argument('error_type', help="""error_type: 'failure' 'pending' 'corrupt_data' 'nomem'""")
1344    p.add_argument(
1345        '-n', '--num', help='the number of commands you want to fail', type=int)
1346    p.add_argument(
1347        '-q', '--queue-depth', help='the queue depth at which to trigger the error', type=int)
1348    p.add_argument(
1349        '-o', '--corrupt-offset', help='the offset in bytes to xor with corrupt_value', type=int)
1350    p.add_argument(
1351        '-v', '--corrupt-value', help='the value for xor (1-255, 0 is invalid)', type=int)
1352    p.set_defaults(func=bdev_error_inject_error)
1353
1354    def bdev_nvme_apply_firmware(args):
1355        print_dict(rpc.bdev.bdev_nvme_apply_firmware(args.client,
1356                                                     bdev_name=args.bdev_name,
1357                                                     filename=args.filename))
1358
1359    p = subparsers.add_parser('bdev_nvme_apply_firmware', help='Download and commit firmware to NVMe device')
1360    p.add_argument('filename', help='filename of the firmware to download')
1361    p.add_argument('bdev_name', help='name of the NVMe device')
1362    p.set_defaults(func=bdev_nvme_apply_firmware)
1363
1364    def bdev_nvme_get_transport_statistics(args):
1365        print_dict(rpc.bdev.bdev_nvme_get_transport_statistics(args.client))
1366
1367    p = subparsers.add_parser('bdev_nvme_get_transport_statistics',
1368                              help='Get bdev_nvme poll group transport statistics')
1369    p.set_defaults(func=bdev_nvme_get_transport_statistics)
1370
1371    def bdev_nvme_get_controller_health_info(args):
1372        print_dict(rpc.bdev.bdev_nvme_get_controller_health_info(args.client,
1373                                                                 name=args.name))
1374
1375    p = subparsers.add_parser('bdev_nvme_get_controller_health_info',
1376                              help='Display health log of the required NVMe bdev controller.')
1377    p.add_argument('-c', '--name', help="Name of the NVMe bdev controller. Example: Nvme0", required=True)
1378    p.set_defaults(func=bdev_nvme_get_controller_health_info)
1379
1380    # iSCSI
1381    def iscsi_set_options(args):
1382        rpc.iscsi.iscsi_set_options(
1383            args.client,
1384            auth_file=args.auth_file,
1385            node_base=args.node_base,
1386            nop_timeout=args.nop_timeout,
1387            nop_in_interval=args.nop_in_interval,
1388            disable_chap=args.disable_chap,
1389            require_chap=args.require_chap,
1390            mutual_chap=args.mutual_chap,
1391            chap_group=args.chap_group,
1392            max_sessions=args.max_sessions,
1393            max_queue_depth=args.max_queue_depth,
1394            max_connections_per_session=args.max_connections_per_session,
1395            default_time2wait=args.default_time2wait,
1396            default_time2retain=args.default_time2retain,
1397            first_burst_length=args.first_burst_length,
1398            immediate_data=args.immediate_data,
1399            error_recovery_level=args.error_recovery_level,
1400            allow_duplicated_isid=args.allow_duplicated_isid,
1401            max_large_datain_per_connection=args.max_large_datain_per_connection,
1402            max_r2t_per_connection=args.max_r2t_per_connection,
1403            pdu_pool_size=args.pdu_pool_size,
1404            immediate_data_pool_size=args.immediate_data_pool_size,
1405            data_out_pool_size=args.data_out_pool_size)
1406
1407    p = subparsers.add_parser('iscsi_set_options',
1408                              help="""Set options of iSCSI subsystem""")
1409    p.add_argument('-f', '--auth-file', help='Path to CHAP shared secret file')
1410    p.add_argument('-b', '--node-base', help='Prefix of the name of iSCSI target node')
1411    p.add_argument('-o', '--nop-timeout', help='Timeout in seconds to nop-in request to the initiator', type=int)
1412    p.add_argument('-n', '--nop-in-interval', help='Time interval in secs between nop-in requests by the target', type=int)
1413    p.add_argument('-d', '--disable-chap', help="""CHAP for discovery session should be disabled.
1414    *** Mutually exclusive with --require-chap""", action='store_true')
1415    p.add_argument('-r', '--require-chap', help="""CHAP for discovery session should be required.
1416    *** Mutually exclusive with --disable-chap""", action='store_true')
1417    p.add_argument('-m', '--mutual-chap', help='CHAP for discovery session should be mutual', action='store_true')
1418    p.add_argument('-g', '--chap-group', help="""Authentication group ID for discovery session.
1419    *** Authentication group must be precreated ***""", type=int)
1420    p.add_argument('-a', '--max-sessions', help='Maximum number of sessions in the host.', type=int)
1421    p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/Os per queue.', type=int)
1422    p.add_argument('-c', '--max-connections-per-session', help='Negotiated parameter, MaxConnections.', type=int)
1423    p.add_argument('-w', '--default-time2wait', help='Negotiated parameter, DefaultTime2Wait.', type=int)
1424    p.add_argument('-v', '--default-time2retain', help='Negotiated parameter, DefaultTime2Retain.', type=int)
1425    p.add_argument('-s', '--first-burst-length', help='Negotiated parameter, FirstBurstLength.', type=int)
1426    p.add_argument('-i', '--immediate-data', help='Negotiated parameter, ImmediateData.', action='store_true')
1427    p.add_argument('-l', '--error-recovery-level', help='Negotiated parameter, ErrorRecoveryLevel', type=int)
1428    p.add_argument('-p', '--allow-duplicated-isid', help='Allow duplicated initiator session ID.', action='store_true')
1429    p.add_argument('-x', '--max-large-datain-per-connection', help='Max number of outstanding split read I/Os per connection', type=int)
1430    p.add_argument('-k', '--max-r2t-per-connection', help='Max number of outstanding R2Ts per connection', type=int)
1431    p.add_argument('-u', '--pdu-pool-size', help='Number of PDUs in the pool', type=int)
1432    p.add_argument('-j', '--immediate-data-pool-size', help='Number of immediate data buffers in the pool', type=int)
1433    p.add_argument('-z', '--data-out-pool-size', help='Number of data out buffers in the pool', type=int)
1434    p.set_defaults(func=iscsi_set_options)
1435
1436    def iscsi_set_discovery_auth(args):
1437        rpc.iscsi.iscsi_set_discovery_auth(
1438            args.client,
1439            disable_chap=args.disable_chap,
1440            require_chap=args.require_chap,
1441            mutual_chap=args.mutual_chap,
1442            chap_group=args.chap_group)
1443
1444    p = subparsers.add_parser('iscsi_set_discovery_auth',
1445                              help="""Set CHAP authentication for discovery session.""")
1446    p.add_argument('-d', '--disable-chap', help="""CHAP for discovery session should be disabled.
1447    *** Mutually exclusive with --require-chap""", action='store_true')
1448    p.add_argument('-r', '--require-chap', help="""CHAP for discovery session should be required.
1449    *** Mutually exclusive with --disable-chap""", action='store_true')
1450    p.add_argument('-m', '--mutual-chap', help='CHAP for discovery session should be mutual', action='store_true')
1451    p.add_argument('-g', '--chap-group', help="""Authentication group ID for discovery session.
1452    *** Authentication group must be precreated ***""", type=int)
1453    p.set_defaults(func=iscsi_set_discovery_auth)
1454
1455    def iscsi_create_auth_group(args):
1456        secrets = None
1457        if args.secrets:
1458            secrets = [dict(u.split(":") for u in a.split(" ")) for a in args.secrets.split(",")]
1459
1460        rpc.iscsi.iscsi_create_auth_group(args.client, tag=args.tag, secrets=secrets)
1461
1462    p = subparsers.add_parser('iscsi_create_auth_group',
1463                              help='Create authentication group for CHAP authentication.')
1464    p.add_argument('tag', help='Authentication group tag (unique, integer > 0).', type=int)
1465    p.add_argument('-c', '--secrets', help="""Comma-separated list of CHAP secrets
1466<user:user_name secret:chap_secret muser:mutual_user_name msecret:mutual_chap_secret> enclosed in quotes.
1467Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 msecret:ms2'""")
1468    p.set_defaults(func=iscsi_create_auth_group)
1469
1470    def iscsi_delete_auth_group(args):
1471        rpc.iscsi.iscsi_delete_auth_group(args.client, tag=args.tag)
1472
1473    p = subparsers.add_parser('iscsi_delete_auth_group',
1474                              help='Delete an authentication group.')
1475    p.add_argument('tag', help='Authentication group tag', type=int)
1476    p.set_defaults(func=iscsi_delete_auth_group)
1477
1478    def iscsi_auth_group_add_secret(args):
1479        rpc.iscsi.iscsi_auth_group_add_secret(
1480            args.client,
1481            tag=args.tag,
1482            user=args.user,
1483            secret=args.secret,
1484            muser=args.muser,
1485            msecret=args.msecret)
1486
1487    p = subparsers.add_parser('iscsi_auth_group_add_secret',
1488                              help='Add a secret to an authentication group.')
1489    p.add_argument('tag', help='Authentication group tag', type=int)
1490    p.add_argument('-u', '--user', help='User name for one-way CHAP authentication', required=True)
1491    p.add_argument('-s', '--secret', help='Secret for one-way CHAP authentication', required=True)
1492    p.add_argument('-m', '--muser', help='User name for mutual CHAP authentication')
1493    p.add_argument('-r', '--msecret', help='Secret for mutual CHAP authentication')
1494    p.set_defaults(func=iscsi_auth_group_add_secret)
1495
1496    def iscsi_auth_group_remove_secret(args):
1497        rpc.iscsi.iscsi_auth_group_remove_secret(args.client, tag=args.tag, user=args.user)
1498
1499    p = subparsers.add_parser('iscsi_auth_group_remove_secret',
1500                              help='Remove a secret from an authentication group.')
1501    p.add_argument('tag', help='Authentication group tag', type=int)
1502    p.add_argument('-u', '--user', help='User name for one-way CHAP authentication', required=True)
1503    p.set_defaults(func=iscsi_auth_group_remove_secret)
1504
1505    def iscsi_get_auth_groups(args):
1506        print_dict(rpc.iscsi.iscsi_get_auth_groups(args.client))
1507
1508    p = subparsers.add_parser('iscsi_get_auth_groups',
1509                              help='Display current authentication group configuration')
1510    p.set_defaults(func=iscsi_get_auth_groups)
1511
1512    def iscsi_get_portal_groups(args):
1513        print_dict(rpc.iscsi.iscsi_get_portal_groups(args.client))
1514
1515    p = subparsers.add_parser('iscsi_get_portal_groups', help='Display current portal group configuration')
1516    p.set_defaults(func=iscsi_get_portal_groups)
1517
1518    def iscsi_get_initiator_groups(args):
1519        print_dict(rpc.iscsi.iscsi_get_initiator_groups(args.client))
1520
1521    p = subparsers.add_parser('iscsi_get_initiator_groups',
1522                              help='Display current initiator group configuration')
1523    p.set_defaults(func=iscsi_get_initiator_groups)
1524
1525    def iscsi_get_target_nodes(args):
1526        print_dict(rpc.iscsi.iscsi_get_target_nodes(args.client))
1527
1528    p = subparsers.add_parser('iscsi_get_target_nodes', help='Display target nodes')
1529    p.set_defaults(func=iscsi_get_target_nodes)
1530
1531    def iscsi_enable_histogram(args):
1532        rpc.iscsi.iscsi_enable_histogram(args.client, name=args.name, enable=args.enable)
1533
1534    p = subparsers.add_parser('iscsi_enable_histogram',
1535                              help='Enable or disable histogram for specified iscsi target')
1536    p.add_argument('-e', '--enable', default=True, dest='enable', action='store_true', help='Enable histograms on specified iscsi target')
1537    p.add_argument('-d', '--disable', dest='enable', action='store_false', help='Disable histograms on specified iscsi target')
1538    p.add_argument('name', help='iscsi target name')
1539    p.set_defaults(func=iscsi_enable_histogram)
1540
1541    def iscsi_get_histogram(args):
1542        print_dict(rpc.iscsi.iscsi_get_histogram(args.client, name=args.name))
1543
1544    p = subparsers.add_parser('iscsi_get_histogram',
1545                              help='Get histogram for specified iscsi target')
1546    p.add_argument('name', help='target name')
1547    p.set_defaults(func=iscsi_get_histogram)
1548
1549    def iscsi_create_target_node(args):
1550        luns = []
1551        for u in args.bdev_name_id_pairs.strip().split(" "):
1552            bdev_name, lun_id = u.split(":")
1553            luns.append({"bdev_name": bdev_name, "lun_id": int(lun_id)})
1554
1555        pg_ig_maps = []
1556        for u in args.pg_ig_mappings.strip().split(" "):
1557            pg, ig = u.split(":")
1558            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1559
1560        rpc.iscsi.iscsi_create_target_node(
1561            args.client,
1562            luns=luns,
1563            pg_ig_maps=pg_ig_maps,
1564            name=args.name,
1565            alias_name=args.alias_name,
1566            queue_depth=args.queue_depth,
1567            chap_group=args.chap_group,
1568            disable_chap=args.disable_chap,
1569            require_chap=args.require_chap,
1570            mutual_chap=args.mutual_chap,
1571            header_digest=args.header_digest,
1572            data_digest=args.data_digest)
1573
1574    p = subparsers.add_parser('iscsi_create_target_node', help='Add a target node')
1575    p.add_argument('name', help='Target node name (ASCII)')
1576    p.add_argument('alias_name', help='Target node alias name (ASCII)')
1577    p.add_argument('bdev_name_id_pairs', help="""Whitespace-separated list of <bdev name:LUN ID> pairs enclosed
1578    in quotes.  Format:  'bdev_name0:id0 bdev_name1:id1' etc
1579    Example: 'Malloc0:0 Malloc1:1 Malloc5:2'
1580    *** The bdevs must pre-exist ***
1581    *** LUN0 (id = 0) is required ***
1582    *** bdevs names cannot contain space or colon characters ***""")
1583    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1584    Whitespace separated, quoted, mapping defined with colon
1585    separated list of "tags" (int > 0)
1586    Example: '1:1 2:2 2:1'
1587    *** The Portal/Initiator Groups must be precreated ***""")
1588    p.add_argument('queue_depth', help='Desired target queue depth', type=int)
1589    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this target node.
1590    *** Authentication group must be precreated ***""", type=int)
1591    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this target node.
1592    *** Mutually exclusive with --require-chap ***""", action='store_true')
1593    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this target node.
1594    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1595    p.add_argument(
1596        '-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.', action='store_true')
1597    p.add_argument('-H', '--header-digest',
1598                   help='Header Digest should be required for this target node.', action='store_true')
1599    p.add_argument('-D', '--data-digest',
1600                   help='Data Digest should be required for this target node.', action='store_true')
1601    p.set_defaults(func=iscsi_create_target_node)
1602
1603    def iscsi_target_node_add_lun(args):
1604        rpc.iscsi.iscsi_target_node_add_lun(
1605            args.client,
1606            name=args.name,
1607            bdev_name=args.bdev_name,
1608            lun_id=args.lun_id)
1609
1610    p = subparsers.add_parser('iscsi_target_node_add_lun',
1611                              help='Add LUN to the target node')
1612    p.add_argument('name', help='Target node name (ASCII)')
1613    p.add_argument('bdev_name', help="""bdev name enclosed in quotes.
1614    *** bdev name cannot contain space or colon characters ***""")
1615    p.add_argument('-i', dest='lun_id', help="""LUN ID (integer >= 0)
1616    *** If LUN ID is omitted or -1, the lowest free one is assigned ***""", type=int)
1617    p.set_defaults(func=iscsi_target_node_add_lun)
1618
1619    def iscsi_target_node_set_auth(args):
1620        rpc.iscsi.iscsi_target_node_set_auth(
1621            args.client,
1622            name=args.name,
1623            chap_group=args.chap_group,
1624            disable_chap=args.disable_chap,
1625            require_chap=args.require_chap,
1626            mutual_chap=args.mutual_chap)
1627
1628    p = subparsers.add_parser('iscsi_target_node_set_auth',
1629                              help='Set CHAP authentication for the target node')
1630    p.add_argument('name', help='Target node name (ASCII)')
1631    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this target node.
1632    *** Authentication group must be precreated ***""", type=int)
1633    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this target node.
1634    *** Mutually exclusive with --require-chap ***""", action='store_true')
1635    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this target node.
1636    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1637    p.add_argument('-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.',
1638                   action='store_true')
1639    p.set_defaults(func=iscsi_target_node_set_auth)
1640
1641    def iscsi_target_node_add_pg_ig_maps(args):
1642        pg_ig_maps = []
1643        for u in args.pg_ig_mappings.strip().split(" "):
1644            pg, ig = u.split(":")
1645            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1646        rpc.iscsi.iscsi_target_node_add_pg_ig_maps(
1647            args.client,
1648            pg_ig_maps=pg_ig_maps,
1649            name=args.name)
1650
1651    p = subparsers.add_parser('iscsi_target_node_add_pg_ig_maps',
1652                              help='Add PG-IG maps to the target node')
1653    p.add_argument('name', help='Target node name (ASCII)')
1654    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1655    Whitespace separated, quoted, mapping defined with colon
1656    separated list of "tags" (int > 0)
1657    Example: '1:1 2:2 2:1'
1658    *** The Portal/Initiator Groups must be precreated ***""")
1659    p.set_defaults(func=iscsi_target_node_add_pg_ig_maps)
1660
1661    def iscsi_target_node_remove_pg_ig_maps(args):
1662        pg_ig_maps = []
1663        for u in args.pg_ig_mappings.strip().split(" "):
1664            pg, ig = u.split(":")
1665            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1666        rpc.iscsi.iscsi_target_node_remove_pg_ig_maps(
1667            args.client, pg_ig_maps=pg_ig_maps, name=args.name)
1668
1669    p = subparsers.add_parser('iscsi_target_node_remove_pg_ig_maps',
1670                              help='Delete PG-IG maps from the target node')
1671    p.add_argument('name', help='Target node name (ASCII)')
1672    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1673    Whitespace separated, quoted, mapping defined with colon
1674    separated list of "tags" (int > 0)
1675    Example: '1:1 2:2 2:1'
1676    *** The Portal/Initiator Groups must be precreated ***""")
1677    p.set_defaults(func=iscsi_target_node_remove_pg_ig_maps)
1678
1679    def iscsi_target_node_set_redirect(args):
1680        rpc.iscsi.iscsi_target_node_set_redirect(
1681            args.client,
1682            name=args.name,
1683            pg_tag=args.pg_tag,
1684            redirect_host=args.redirect_host,
1685            redirect_port=args.redirect_port)
1686
1687    p = subparsers.add_parser('iscsi_target_node_set_redirect',
1688                              help="""Update redirect portal of the public portal group for the target node.
1689    Omit redirect host and port to clear previously set redirect settings.""")
1690    p.add_argument('name', help='Target node name (ASCII)')
1691    p.add_argument('pg_tag', help='Portal group tag (unique, integer > 0)', type=int)
1692    p.add_argument('-a', '--redirect-host', help='Numeric IP address for redirect portal')
1693    p.add_argument('-p', '--redirect-port', help='Numeric TCP port for redirect portal')
1694    p.set_defaults(func=iscsi_target_node_set_redirect)
1695
1696    def iscsi_target_node_request_logout(args):
1697        rpc.iscsi.iscsi_target_node_request_logout(
1698            args.client,
1699            name=args.name,
1700            pg_tag=args.pg_tag)
1701
1702    p = subparsers.add_parser('iscsi_target_node_request_logout',
1703                              help="""For the target node, request connections whose portal group tag
1704    match to logout, or request all connections if portal group tag is omitted.""")
1705    p.add_argument('name', help='Target node name (ASCII)')
1706    p.add_argument('-t', '--pg-tag', help='Portal group tag (unique, integer > 0)', type=int)
1707    p.set_defaults(func=iscsi_target_node_request_logout)
1708
1709    def iscsi_create_portal_group(args):
1710        portals = []
1711        for p in args.portal_list.strip().split(' '):
1712            ip, separator, port_cpumask = p.rpartition(':')
1713            split_port_cpumask = port_cpumask.split('@')
1714            if len(split_port_cpumask) == 1:
1715                port = port_cpumask
1716                portals.append({'host': ip, 'port': port})
1717            else:
1718                port = split_port_cpumask[0]
1719                cpumask = split_port_cpumask[1]
1720                portals.append({'host': ip, 'port': port})
1721                print("WARNING: Specifying a portal group with a CPU mask is no longer supported. Ignoring it.")
1722        rpc.iscsi.iscsi_create_portal_group(
1723            args.client,
1724            portals=portals,
1725            tag=args.tag,
1726            private=args.private,
1727            wait=args.wait)
1728
1729    p = subparsers.add_parser('iscsi_create_portal_group',
1730                              help='Add a portal group')
1731    p.add_argument(
1732        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1733    p.add_argument('portal_list', help="""List of portals in host:port format, separated by whitespace
1734    Example: '192.168.100.100:3260 192.168.100.100:3261 192.168.100.100:3262""")
1735    p.add_argument('-p', '--private', help="""Public (false) or private (true) portal group.
1736    Private portal groups do not have their portals returned by a discovery session. A public
1737    portal group may optionally specify a redirect portal for non-discovery logins. This redirect
1738    portal must be from a private portal group.""", action='store_true')
1739    p.add_argument('-w', '--wait', help="""Do not listening on portals until it is started explicitly.
1740    One major iSCSI initiator may not retry login once it failed. Hence for such initiator, listening
1741    on portals should be allowed after all associated target nodes are created.""", action='store_true')
1742    p.set_defaults(func=iscsi_create_portal_group)
1743
1744    def iscsi_start_portal_group(args):
1745        rpc.iscsi.iscsi_start_portal_group(args.client, tag=args.tag)
1746
1747    p = subparsers.add_parser('iscsi_start_portal_group',
1748                              help='Start listening on portals if it is not started yet.')
1749    p.add_argument(
1750        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1751    p.set_defaults(func=iscsi_start_portal_group)
1752
1753    def iscsi_create_initiator_group(args):
1754        initiators = []
1755        netmasks = []
1756        for i in args.initiator_list.strip().split(' '):
1757            initiators.append(i)
1758        for n in args.netmask_list.strip().split(' '):
1759            netmasks.append(n)
1760        rpc.iscsi.iscsi_create_initiator_group(
1761            args.client,
1762            tag=args.tag,
1763            initiators=initiators,
1764            netmasks=netmasks)
1765
1766    p = subparsers.add_parser('iscsi_create_initiator_group',
1767                              help='Add an initiator group')
1768    p.add_argument(
1769        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1770    p.add_argument('initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1771    enclosed in quotes.  Example: 'ANY' or 'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""")
1772    p.add_argument('netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1773    Example: '255.255.0.0 255.248.0.0' etc""")
1774    p.set_defaults(func=iscsi_create_initiator_group)
1775
1776    def iscsi_initiator_group_add_initiators(args):
1777        initiators = None
1778        netmasks = None
1779        if args.initiator_list:
1780            initiators = []
1781            for i in args.initiator_list.strip().split(' '):
1782                initiators.append(i)
1783        if args.netmask_list:
1784            netmasks = []
1785            for n in args.netmask_list.strip().split(' '):
1786                netmasks.append(n)
1787        rpc.iscsi.iscsi_initiator_group_add_initiators(
1788            args.client,
1789            tag=args.tag,
1790            initiators=initiators,
1791            netmasks=netmasks)
1792
1793    p = subparsers.add_parser('iscsi_initiator_group_add_initiators',
1794                              help='Add initiators to an existing initiator group')
1795    p.add_argument(
1796        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1797    p.add_argument('-n', dest='initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1798    enclosed in quotes.  This parameter can be omitted.  Example: 'ANY' or
1799    'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""")
1800    p.add_argument('-m', dest='netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1801    This parameter can be omitted.  Example: '255.255.0.0 255.248.0.0' etc""")
1802    p.set_defaults(func=iscsi_initiator_group_add_initiators)
1803
1804    def iscsi_initiator_group_remove_initiators(args):
1805        initiators = None
1806        netmasks = None
1807        if args.initiator_list:
1808            initiators = []
1809            for i in args.initiator_list.strip().split(' '):
1810                initiators.append(i)
1811        if args.netmask_list:
1812            netmasks = []
1813            for n in args.netmask_list.strip().split(' '):
1814                netmasks.append(n)
1815        rpc.iscsi.iscsi_initiator_group_remove_initiators(
1816            args.client,
1817            tag=args.tag,
1818            initiators=initiators,
1819            netmasks=netmasks)
1820
1821    p = subparsers.add_parser('iscsi_initiator_group_remove_initiators',
1822                              help='Delete initiators from an existing initiator group')
1823    p.add_argument(
1824        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1825    p.add_argument('-n', dest='initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1826    enclosed in quotes.  This parameter can be omitted.  Example: 'ANY' or
1827    'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""")
1828    p.add_argument('-m', dest='netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1829    This parameter can be omitted.  Example: '255.255.0.0 255.248.0.0' etc""")
1830    p.set_defaults(func=iscsi_initiator_group_remove_initiators)
1831
1832    def iscsi_delete_target_node(args):
1833        rpc.iscsi.iscsi_delete_target_node(
1834            args.client, target_node_name=args.target_node_name)
1835
1836    p = subparsers.add_parser('iscsi_delete_target_node',
1837                              help='Delete a target node')
1838    p.add_argument('target_node_name',
1839                   help='Target node name to be deleted. Example: iqn.2016-06.io.spdk:disk1.')
1840    p.set_defaults(func=iscsi_delete_target_node)
1841
1842    def iscsi_delete_portal_group(args):
1843        rpc.iscsi.iscsi_delete_portal_group(args.client, tag=args.tag)
1844
1845    p = subparsers.add_parser('iscsi_delete_portal_group',
1846                              help='Delete a portal group')
1847    p.add_argument(
1848        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1849    p.set_defaults(func=iscsi_delete_portal_group)
1850
1851    def iscsi_delete_initiator_group(args):
1852        rpc.iscsi.iscsi_delete_initiator_group(args.client, tag=args.tag)
1853
1854    p = subparsers.add_parser('iscsi_delete_initiator_group',
1855                              help='Delete an initiator group')
1856    p.add_argument(
1857        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1858    p.set_defaults(func=iscsi_delete_initiator_group)
1859
1860    def iscsi_portal_group_set_auth(args):
1861        rpc.iscsi.iscsi_portal_group_set_auth(
1862            args.client,
1863            tag=args.tag,
1864            chap_group=args.chap_group,
1865            disable_chap=args.disable_chap,
1866            require_chap=args.require_chap,
1867            mutual_chap=args.mutual_chap)
1868
1869    p = subparsers.add_parser('iscsi_portal_group_set_auth',
1870                              help='Set CHAP authentication for discovery sessions specific for the portal group')
1871    p.add_argument('tag', help='Portal group tag (unique, integer > 0)', type=int)
1872    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this portal group.
1873    *** Authentication group must be precreated ***""", type=int)
1874    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this portal group.
1875    *** Mutually exclusive with --require-chap ***""", action='store_true')
1876    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this portal group.
1877    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1878    p.add_argument('-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.',
1879                   action='store_true')
1880    p.set_defaults(func=iscsi_portal_group_set_auth)
1881
1882    def iscsi_get_connections(args):
1883        print_dict(rpc.iscsi.iscsi_get_connections(args.client))
1884
1885    p = subparsers.add_parser('iscsi_get_connections',
1886                              help='Display iSCSI connections')
1887    p.set_defaults(func=iscsi_get_connections)
1888
1889    def iscsi_get_stats(args):
1890        print_dict(rpc.iscsi.iscsi_get_stats(args.client))
1891
1892    p = subparsers.add_parser('iscsi_get_stats',
1893                              help='Display stat information of iSCSI connections.')
1894    p.set_defaults(func=iscsi_get_stats)
1895
1896    def iscsi_get_options(args):
1897        print_dict(rpc.iscsi.iscsi_get_options(args.client))
1898
1899    p = subparsers.add_parser('iscsi_get_options',
1900                              help='Display iSCSI global parameters')
1901    p.set_defaults(func=iscsi_get_options)
1902
1903    def scsi_get_devices(args):
1904        print_dict(rpc.iscsi.scsi_get_devices(args.client))
1905
1906    p = subparsers.add_parser('scsi_get_devices', help='Display SCSI devices')
1907    p.set_defaults(func=scsi_get_devices)
1908
1909    # trace
1910    def trace_enable_tpoint_group(args):
1911        rpc.trace.trace_enable_tpoint_group(args.client, name=args.name)
1912
1913    p = subparsers.add_parser('trace_enable_tpoint_group',
1914                              help='enable trace on a specific tpoint group')
1915    p.add_argument(
1916        'name', help="""trace group name we want to enable in tpoint_group_mask.
1917        (for example "bdev" for bdev trace group, "all" for all trace groups).""")
1918    p.set_defaults(func=trace_enable_tpoint_group)
1919
1920    def trace_disable_tpoint_group(args):
1921        rpc.trace.trace_disable_tpoint_group(args.client, name=args.name)
1922
1923    p = subparsers.add_parser('trace_disable_tpoint_group',
1924                              help='disable trace on a specific tpoint group')
1925    p.add_argument(
1926        'name', help="""trace group name we want to disable in tpoint_group_mask.
1927        (for example "bdev" for bdev trace group, "all" for all trace groups).""")
1928    p.set_defaults(func=trace_disable_tpoint_group)
1929
1930    def trace_set_tpoint_mask(args):
1931        rpc.trace.trace_set_tpoint_mask(args.client, name=args.name, tpoint_mask=args.tpoint_mask)
1932
1933    p = subparsers.add_parser('trace_set_tpoint_mask',
1934                              help='enable tracepoint mask on a specific tpoint group')
1935    p.add_argument(
1936        'name', help="""trace group name we want to enable in tpoint_group_mask.
1937        (for example "bdev" for bdev trace group)""")
1938    p.add_argument(
1939        'tpoint_mask', help="""tracepoints to be enabled inside a given trace group.
1940        (for example value of "0x3" will enable only the first two tpoints in this group)""",
1941        type=lambda m: int(m, 16))
1942    p.set_defaults(func=trace_set_tpoint_mask)
1943
1944    def trace_clear_tpoint_mask(args):
1945        rpc.trace.trace_clear_tpoint_mask(args.client, name=args.name, tpoint_mask=args.tpoint_mask)
1946
1947    p = subparsers.add_parser('trace_clear_tpoint_mask',
1948                              help='disable tracepoint mask on a specific tpoint group')
1949    p.add_argument(
1950        'name', help="""trace group name we want to disable in tpoint_group_mask.
1951        (for example "bdev" for bdev trace group)""")
1952    p.add_argument(
1953        'tpoint_mask', help="""tracepoints to be disabled inside a given trace group.
1954        (for example value of "0x3" will disable the first two tpoints in this group)""",
1955        type=lambda m: int(m, 16))
1956    p.set_defaults(func=trace_clear_tpoint_mask)
1957
1958    def trace_get_tpoint_group_mask(args):
1959        print_dict(rpc.trace.trace_get_tpoint_group_mask(args.client))
1960
1961    p = subparsers.add_parser('trace_get_tpoint_group_mask', help='get trace point group mask')
1962    p.set_defaults(func=trace_get_tpoint_group_mask)
1963
1964    def trace_get_info(args):
1965        print_dict(rpc.trace.trace_get_info(args.client))
1966
1967    p = subparsers.add_parser('trace_get_info',
1968                              help='get name of shared memory file and list of the available trace point groups')
1969    p.set_defaults(func=trace_get_info)
1970
1971    # log
1972    def log_set_flag(args):
1973        rpc.log.log_set_flag(args.client, flag=args.flag)
1974
1975    p = subparsers.add_parser('log_set_flag', help='set log flag')
1976    p.add_argument(
1977        'flag', help='log flag we want to set. (for example "nvme").')
1978    p.set_defaults(func=log_set_flag)
1979
1980    def log_clear_flag(args):
1981        rpc.log.log_clear_flag(args.client, flag=args.flag)
1982
1983    p = subparsers.add_parser('log_clear_flag', help='clear log flag')
1984    p.add_argument(
1985        'flag', help='log flag we want to clear. (for example "nvme").')
1986    p.set_defaults(func=log_clear_flag)
1987
1988    def log_get_flags(args):
1989        print_dict(rpc.log.log_get_flags(args.client))
1990
1991    p = subparsers.add_parser('log_get_flags', help='get log flags')
1992    p.set_defaults(func=log_get_flags)
1993
1994    def log_set_level(args):
1995        rpc.log.log_set_level(args.client, level=args.level)
1996
1997    p = subparsers.add_parser('log_set_level', help='set log level')
1998    p.add_argument('level', help='log level we want to set. (for example "DEBUG").')
1999    p.set_defaults(func=log_set_level)
2000
2001    def log_get_level(args):
2002        print_dict(rpc.log.log_get_level(args.client))
2003
2004    p = subparsers.add_parser('log_get_level', help='get log level')
2005    p.set_defaults(func=log_get_level)
2006
2007    def log_set_print_level(args):
2008        rpc.log.log_set_print_level(args.client, level=args.level)
2009
2010    p = subparsers.add_parser('log_set_print_level', help='set log print level')
2011    p.add_argument('level', help='log print level we want to set. (for example "DEBUG").')
2012    p.set_defaults(func=log_set_print_level)
2013
2014    def log_get_print_level(args):
2015        print_dict(rpc.log.log_get_print_level(args.client))
2016
2017    p = subparsers.add_parser('log_get_print_level', help='get log print level')
2018    p.set_defaults(func=log_get_print_level)
2019
2020    # lvol
2021    def bdev_lvol_create_lvstore(args):
2022        # The default unmap clear method may take over 60.0 sec.
2023        if args.timeout is None:
2024            args.client.timeout = 90.0
2025        print_json(rpc.lvol.bdev_lvol_create_lvstore(args.client,
2026                                                     bdev_name=args.bdev_name,
2027                                                     lvs_name=args.lvs_name,
2028                                                     cluster_sz=args.cluster_sz,
2029                                                     clear_method=args.clear_method,
2030                                                     num_md_pages_per_cluster_ratio=args.md_pages_per_cluster_ratio))
2031
2032    p = subparsers.add_parser('bdev_lvol_create_lvstore', help='Add logical volume store on base bdev')
2033    p.add_argument('bdev_name', help='base bdev name')
2034    p.add_argument('lvs_name', help='name for lvol store')
2035    p.add_argument('-c', '--cluster-sz', help='size of cluster (in bytes)', type=int)
2036    p.add_argument('--clear-method', help="""Change clear method for data region.
2037        Available: none, unmap, write_zeroes""")
2038    p.add_argument('-m', '--md-pages-per-cluster-ratio', help='reserved metadata pages for each cluster', type=int)
2039    p.set_defaults(func=bdev_lvol_create_lvstore)
2040
2041    def bdev_lvol_rename_lvstore(args):
2042        rpc.lvol.bdev_lvol_rename_lvstore(args.client,
2043                                          old_name=args.old_name,
2044                                          new_name=args.new_name)
2045
2046    p = subparsers.add_parser('bdev_lvol_rename_lvstore', help='Change logical volume store name')
2047    p.add_argument('old_name', help='old name')
2048    p.add_argument('new_name', help='new name')
2049    p.set_defaults(func=bdev_lvol_rename_lvstore)
2050
2051    def bdev_lvol_grow_lvstore(args):
2052        print_dict(rpc.lvol.bdev_lvol_grow_lvstore(args.client,
2053                                                   uuid=args.uuid,
2054                                                   lvs_name=args.lvs_name))
2055
2056    p = subparsers.add_parser('bdev_lvol_grow_lvstore',
2057                              help='Grow the lvstore size to the underlying bdev size')
2058    p.add_argument('-u', '--uuid', help='lvol store UUID')
2059    p.add_argument('-l', '--lvs-name', help='lvol store name')
2060    p.set_defaults(func=bdev_lvol_grow_lvstore)
2061
2062    def bdev_lvol_create(args):
2063        print_json(rpc.lvol.bdev_lvol_create(args.client,
2064                                             lvol_name=args.lvol_name,
2065                                             size_in_mib=args.size_in_mib,
2066                                             thin_provision=args.thin_provision,
2067                                             clear_method=args.clear_method,
2068                                             uuid=args.uuid,
2069                                             lvs_name=args.lvs_name))
2070
2071    p = subparsers.add_parser('bdev_lvol_create', help='Add a bdev with an logical volume backend')
2072    p.add_argument('-u', '--uuid', help='lvol store UUID')
2073    p.add_argument('-l', '--lvs-name', help='lvol store name')
2074    p.add_argument('-t', '--thin-provision', action='store_true', help='create lvol bdev as thin provisioned')
2075    p.add_argument('-c', '--clear-method', help="""Change default data clusters clear method.
2076        Available: none, unmap, write_zeroes""")
2077    p.add_argument('lvol_name', help='name for this lvol')
2078    p.add_argument('size_in_mib', help='size in MiB for this bdev', type=int)
2079    p.set_defaults(func=bdev_lvol_create)
2080
2081    def bdev_lvol_snapshot(args):
2082        print_json(rpc.lvol.bdev_lvol_snapshot(args.client,
2083                                               lvol_name=args.lvol_name,
2084                                               snapshot_name=args.snapshot_name))
2085
2086    p = subparsers.add_parser('bdev_lvol_snapshot', help='Create a snapshot of an lvol bdev')
2087    p.add_argument('lvol_name', help='lvol bdev name')
2088    p.add_argument('snapshot_name', help='lvol snapshot name')
2089    p.set_defaults(func=bdev_lvol_snapshot)
2090
2091    def bdev_lvol_clone(args):
2092        print_json(rpc.lvol.bdev_lvol_clone(args.client,
2093                                            snapshot_name=args.snapshot_name,
2094                                            clone_name=args.clone_name))
2095
2096    p = subparsers.add_parser('bdev_lvol_clone', help='Create a clone of an lvol snapshot')
2097    p.add_argument('snapshot_name', help='lvol snapshot name')
2098    p.add_argument('clone_name', help='lvol clone name')
2099    p.set_defaults(func=bdev_lvol_clone)
2100
2101    def bdev_lvol_clone_bdev(args):
2102        print_json(rpc.lvol.bdev_lvol_clone_bdev(args.client,
2103                                                 bdev=args.bdev,
2104                                                 lvs_name=args.lvs_name,
2105                                                 clone_name=args.clone_name))
2106
2107    p = subparsers.add_parser('bdev_lvol_clone_bdev',
2108                              help='Create a clone of a non-lvol bdev')
2109    p.add_argument('bdev', help='bdev to clone')
2110    p.add_argument('lvs_name', help='logical volume store name')
2111    p.add_argument('clone_name', help='lvol clone name')
2112    p.set_defaults(func=bdev_lvol_clone_bdev)
2113
2114    def bdev_lvol_rename(args):
2115        rpc.lvol.bdev_lvol_rename(args.client,
2116                                  old_name=args.old_name,
2117                                  new_name=args.new_name)
2118
2119    p = subparsers.add_parser('bdev_lvol_rename', help='Change lvol bdev name')
2120    p.add_argument('old_name', help='lvol bdev name')
2121    p.add_argument('new_name', help='new lvol name')
2122    p.set_defaults(func=bdev_lvol_rename)
2123
2124    def bdev_lvol_inflate(args):
2125        rpc.lvol.bdev_lvol_inflate(args.client,
2126                                   name=args.name)
2127
2128    p = subparsers.add_parser('bdev_lvol_inflate', help='Make thin provisioned lvol a thick provisioned lvol')
2129    p.add_argument('name', help='lvol bdev name')
2130    p.set_defaults(func=bdev_lvol_inflate)
2131
2132    def bdev_lvol_decouple_parent(args):
2133        rpc.lvol.bdev_lvol_decouple_parent(args.client,
2134                                           name=args.name)
2135
2136    p = subparsers.add_parser('bdev_lvol_decouple_parent', help='Decouple parent of lvol')
2137    p.add_argument('name', help='lvol bdev name')
2138    p.set_defaults(func=bdev_lvol_decouple_parent)
2139
2140    def bdev_lvol_resize(args):
2141        rpc.lvol.bdev_lvol_resize(args.client,
2142                                  name=args.name,
2143                                  size_in_mib=args.size_in_mib)
2144
2145    p = subparsers.add_parser('bdev_lvol_resize', help='Resize existing lvol bdev')
2146    p.add_argument('name', help='lvol bdev name')
2147    p.add_argument('size_in_mib', help='new size in MiB for this bdev', type=int)
2148    p.set_defaults(func=bdev_lvol_resize)
2149
2150    def bdev_lvol_set_read_only(args):
2151        rpc.lvol.bdev_lvol_set_read_only(args.client,
2152                                         name=args.name)
2153
2154    p = subparsers.add_parser('bdev_lvol_set_read_only', help='Mark lvol bdev as read only')
2155    p.add_argument('name', help='lvol bdev name')
2156    p.set_defaults(func=bdev_lvol_set_read_only)
2157
2158    def bdev_lvol_delete(args):
2159        rpc.lvol.bdev_lvol_delete(args.client,
2160                                  name=args.name)
2161
2162    p = subparsers.add_parser('bdev_lvol_delete', help='Destroy a logical volume')
2163    p.add_argument('name', help='lvol bdev name')
2164    p.set_defaults(func=bdev_lvol_delete)
2165
2166    def bdev_lvol_start_shallow_copy(args):
2167        print_json(rpc.lvol.bdev_lvol_start_shallow_copy(args.client,
2168                                                         src_lvol_name=args.src_lvol_name,
2169                                                         dst_bdev_name=args.dst_bdev_name))
2170
2171    p = subparsers.add_parser('bdev_lvol_start_shallow_copy',
2172                              help="""Start a shallow copy of an lvol over a given bdev.  The status of the operation
2173    can be obtained with bdev_lvol_check_shallow_copy""")
2174    p.add_argument('src_lvol_name', help='source lvol name')
2175    p.add_argument('dst_bdev_name', help='destination bdev name')
2176    p.set_defaults(func=bdev_lvol_start_shallow_copy)
2177
2178    def bdev_lvol_check_shallow_copy(args):
2179        print_json(rpc.lvol.bdev_lvol_check_shallow_copy(args.client,
2180                                                         operation_id=args.operation_id))
2181
2182    p = subparsers.add_parser('bdev_lvol_check_shallow_copy', help='Get shallow copy status')
2183    p.add_argument('operation_id', help='operation identifier', type=int)
2184    p.set_defaults(func=bdev_lvol_check_shallow_copy)
2185
2186    def bdev_lvol_set_parent(args):
2187        rpc.lvol.bdev_lvol_set_parent(args.client,
2188                                      lvol_name=args.lvol_name,
2189                                      parent_name=args.parent_name)
2190
2191    p = subparsers.add_parser('bdev_lvol_set_parent', help='Set the parent snapshot of a lvol')
2192    p.add_argument('lvol_name', help='lvol name')
2193    p.add_argument('parent_name', help='parent snapshot name')
2194    p.set_defaults(func=bdev_lvol_set_parent)
2195
2196    def bdev_lvol_set_parent_bdev(args):
2197        rpc.lvol.bdev_lvol_set_parent_bdev(args.client,
2198                                           lvol_name=args.lvol_name,
2199                                           parent_name=args.parent_name)
2200
2201    p = subparsers.add_parser('bdev_lvol_set_parent_bdev', help='Set the parent external snapshot of a lvol')
2202    p.add_argument('lvol_name', help='lvol name')
2203    p.add_argument('parent_name', help='parent external snapshot name')
2204    p.set_defaults(func=bdev_lvol_set_parent_bdev)
2205
2206    def bdev_lvol_delete_lvstore(args):
2207        rpc.lvol.bdev_lvol_delete_lvstore(args.client,
2208                                          uuid=args.uuid,
2209                                          lvs_name=args.lvs_name)
2210
2211    p = subparsers.add_parser('bdev_lvol_delete_lvstore', help='Destroy an logical volume store')
2212    p.add_argument('-u', '--uuid', help='lvol store UUID')
2213    p.add_argument('-l', '--lvs-name', help='lvol store name')
2214    p.set_defaults(func=bdev_lvol_delete_lvstore)
2215
2216    def bdev_lvol_get_lvstores(args):
2217        print_dict(rpc.lvol.bdev_lvol_get_lvstores(args.client,
2218                                                   uuid=args.uuid,
2219                                                   lvs_name=args.lvs_name))
2220
2221    p = subparsers.add_parser('bdev_lvol_get_lvstores', help='Display current logical volume store list')
2222    p.add_argument('-u', '--uuid', help='lvol store UUID')
2223    p.add_argument('-l', '--lvs-name', help='lvol store name')
2224    p.set_defaults(func=bdev_lvol_get_lvstores)
2225
2226    def bdev_lvol_get_lvols(args):
2227        print_dict(rpc.lvol.bdev_lvol_get_lvols(args.client,
2228                                                lvs_uuid=args.lvs_uuid,
2229                                                lvs_name=args.lvs_name))
2230
2231    p = subparsers.add_parser('bdev_lvol_get_lvols', help='Display current logical volume list')
2232    p.add_argument('-u', '--lvs-uuid', help='only lvols in  lvol store UUID')
2233    p.add_argument('-l', '--lvs-name', help='only lvols in lvol store name')
2234    p.set_defaults(func=bdev_lvol_get_lvols)
2235
2236    def bdev_raid_set_options(args):
2237        rpc.bdev.bdev_raid_set_options(args.client,
2238                                       process_window_size_kb=args.process_window_size_kb,
2239                                       process_max_bandwidth_mb_sec=args.process_max_bandwidth_mb_sec)
2240
2241    p = subparsers.add_parser('bdev_raid_set_options',
2242                              help='Set options for bdev raid.')
2243    p.add_argument('-w', '--process-window-size-kb', type=int,
2244                   help="Background process (e.g. rebuild) window size in KiB")
2245    p.add_argument('-b', '--process-max-bandwidth-mb-sec', type=int,
2246                   help="Background process (e.g. rebuild) maximum bandwidth in MiB/Sec")
2247
2248    p.set_defaults(func=bdev_raid_set_options)
2249
2250    def bdev_raid_get_bdevs(args):
2251        print_json(rpc.bdev.bdev_raid_get_bdevs(args.client,
2252                                                category=args.category))
2253
2254    p = subparsers.add_parser('bdev_raid_get_bdevs',
2255                              help="""This is used to list all the raid bdev details based on the input category
2256    requested. Category should be one of 'all', 'online', 'configuring' or 'offline'. 'all' means all the raid bdevs whether
2257    they are online or configuring or offline. 'online' is the raid bdev which is registered with bdev layer. 'configuring'
2258    is the raid bdev which does not have full configuration discovered yet. 'offline' is the raid bdev which is not registered
2259    with bdev as of now and it has encountered any error or user has requested to offline the raid bdev""")
2260    p.add_argument('category', help='all or online or configuring or offline')
2261    p.set_defaults(func=bdev_raid_get_bdevs)
2262
2263    def bdev_raid_create(args):
2264        base_bdevs = []
2265        for u in args.base_bdevs.strip().split():
2266            base_bdevs.append(u)
2267
2268        rpc.bdev.bdev_raid_create(args.client,
2269                                  name=args.name,
2270                                  strip_size_kb=args.strip_size_kb,
2271                                  raid_level=args.raid_level,
2272                                  base_bdevs=base_bdevs,
2273                                  uuid=args.uuid,
2274                                  superblock=args.superblock)
2275    p = subparsers.add_parser('bdev_raid_create', help='Create new raid bdev')
2276    p.add_argument('-n', '--name', help='raid bdev name', required=True)
2277    p.add_argument('-z', '--strip-size-kb', help='strip size in KB', type=int)
2278    p.add_argument('-r', '--raid-level', help='raid level, raid0, raid1 and a special level concat are supported', required=True)
2279    p.add_argument('-b', '--base-bdevs', help='base bdevs name, whitespace separated list in quotes', required=True)
2280    p.add_argument('--uuid', help='UUID for this raid bdev')
2281    p.add_argument('-s', '--superblock', help='information about raid bdev will be stored in superblock on each base bdev, '
2282                                              'disabled by default due to backward compatibility', action='store_true')
2283    p.set_defaults(func=bdev_raid_create)
2284
2285    def bdev_raid_delete(args):
2286        rpc.bdev.bdev_raid_delete(args.client,
2287                                  name=args.name)
2288    p = subparsers.add_parser('bdev_raid_delete', help='Delete existing raid bdev')
2289    p.add_argument('name', help='raid bdev name')
2290    p.set_defaults(func=bdev_raid_delete)
2291
2292    def bdev_raid_add_base_bdev(args):
2293        rpc.bdev.bdev_raid_add_base_bdev(args.client,
2294                                         raid_bdev=args.raid_bdev,
2295                                         base_bdev=args.base_bdev)
2296    p = subparsers.add_parser('bdev_raid_add_base_bdev', help='Add base bdev to existing raid bdev')
2297    p.add_argument('raid_bdev', help='raid bdev name')
2298    p.add_argument('base_bdev', help='base bdev name')
2299    p.set_defaults(func=bdev_raid_add_base_bdev)
2300
2301    def bdev_raid_remove_base_bdev(args):
2302        rpc.bdev.bdev_raid_remove_base_bdev(args.client,
2303                                            name=args.name)
2304    p = subparsers.add_parser('bdev_raid_remove_base_bdev', help='Remove base bdev from existing raid bdev')
2305    p.add_argument('name', help='base bdev name')
2306    p.set_defaults(func=bdev_raid_remove_base_bdev)
2307
2308    # split
2309    def bdev_split_create(args):
2310        print_array(rpc.bdev.bdev_split_create(args.client,
2311                                               base_bdev=args.base_bdev,
2312                                               split_count=args.split_count,
2313                                               split_size_mb=args.split_size_mb))
2314
2315    p = subparsers.add_parser('bdev_split_create',
2316                              help="""Add given disk name to split config. If bdev with base_name
2317    name exist the split bdevs will be created right away, if not split bdevs will be created when base bdev became
2318    available (during examination process).""")
2319    p.add_argument('base_bdev', help='base bdev name')
2320    p.add_argument('-s', '--split-size-mb', help='size in MiB for each bdev', type=int)
2321    p.add_argument('split_count', help="""Optional - number of split bdevs to create. Total size * split_count must not
2322    exceed the base bdev size.""", type=int)
2323    p.set_defaults(func=bdev_split_create)
2324
2325    def bdev_split_delete(args):
2326        rpc.bdev.bdev_split_delete(args.client,
2327                                   base_bdev=args.base_bdev)
2328
2329    p = subparsers.add_parser('bdev_split_delete', help="""Delete split config with all created splits.""")
2330    p.add_argument('base_bdev', help='base bdev name')
2331    p.set_defaults(func=bdev_split_delete)
2332
2333    # ftl
2334    def bdev_ftl_create(args):
2335        print_dict(rpc.bdev.bdev_ftl_create(args.client,
2336                                            name=args.name,
2337                                            base_bdev=args.base_bdev,
2338                                            uuid=args.uuid,
2339                                            cache=args.cache,
2340                                            overprovisioning=args.overprovisioning,
2341                                            l2p_dram_limit=args.l2p_dram_limit,
2342                                            core_mask=args.core_mask,
2343                                            fast_shutdown=args.fast_shutdown))
2344
2345    p = subparsers.add_parser('bdev_ftl_create', help='Add FTL bdev')
2346    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2347    p.add_argument('-d', '--base-bdev', help='Name of bdev used as underlying device',
2348                   required=True)
2349    p.add_argument('-u', '--uuid', help='UUID of restored bdev (not applicable when creating new '
2350                   'instance): e.g. b286d19a-0059-4709-abcd-9f7732b1567d (optional)')
2351    p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache',
2352                   required=True)
2353    p.add_argument('--overprovisioning', help='Percentage of device used for relocation, not exposed'
2354                   ' to user (optional); default 20', type=int)
2355    p.add_argument('--l2p-dram-limit', help='l2p size that could reside in DRAM (optional); default 2048',
2356                   type=int)
2357    p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, '
2358                   'by default core thread will be set to the main application core (optional)')
2359    p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true')
2360    p.set_defaults(func=bdev_ftl_create)
2361
2362    def bdev_ftl_load(args):
2363        print_dict(rpc.bdev.bdev_ftl_load(args.client,
2364                                          name=args.name,
2365                                          base_bdev=args.base_bdev,
2366                                          uuid=args.uuid,
2367                                          cache=args.cache,
2368                                          overprovisioning=args.overprovisioning,
2369                                          l2p_dram_limit=args.l2p_dram_limit,
2370                                          core_mask=args.core_mask,
2371                                          fast_shutdown=args.fast_shutdown))
2372
2373    p = subparsers.add_parser('bdev_ftl_load', help='Load FTL bdev')
2374    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2375    p.add_argument('-d', '--base-bdev', help='Name of bdev used as underlying device',
2376                   required=True)
2377    p.add_argument('-u', '--uuid', help='UUID of restored bdev', required=True)
2378    p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache',
2379                   required=True)
2380    p.add_argument('--overprovisioning', help='Percentage of device used for relocation, not exposed'
2381                   ' to user (optional); default 20', type=int)
2382    p.add_argument('--l2p-dram-limit', help='l2p size that could reside in DRAM (optional); default 2048',
2383                   type=int)
2384    p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, '
2385                   'by default core thread will be set to the main application core (optional)')
2386    p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true')
2387    p.set_defaults(func=bdev_ftl_load)
2388
2389    def bdev_ftl_unload(args):
2390        print_dict(rpc.bdev.bdev_ftl_unload(args.client, name=args.name, fast_shutdown=args.fast_shutdown))
2391
2392    p = subparsers.add_parser('bdev_ftl_unload', help='Unload FTL bdev')
2393    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2394    p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true')
2395    p.set_defaults(func=bdev_ftl_unload)
2396
2397    def bdev_ftl_delete(args):
2398        print_dict(rpc.bdev.bdev_ftl_delete(args.client, name=args.name, fast_shutdown=args.fast_shutdown))
2399
2400    p = subparsers.add_parser('bdev_ftl_delete', help='Delete FTL bdev')
2401    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2402    p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true')
2403    p.set_defaults(func=bdev_ftl_delete)
2404
2405    def bdev_ftl_unmap(args):
2406        print_dict(rpc.bdev.bdev_ftl_unmap(args.client, name=args.name,
2407                                           lba=args.lba,
2408                                           num_blocks=args.num_blocks))
2409
2410    p = subparsers.add_parser('bdev_ftl_unmap', help='FTL unmap')
2411    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2412    p.add_argument('--lba', help='start LBA', required=True, type=int)
2413    p.add_argument('--num-blocks', help='num blocks', required=True, type=int)
2414    p.set_defaults(func=bdev_ftl_unmap)
2415
2416    def bdev_ftl_get_stats(args):
2417        print_dict(rpc.bdev.bdev_ftl_get_stats(args.client, name=args.name))
2418
2419    p = subparsers.add_parser('bdev_ftl_get_stats', help='print ftl stats')
2420    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2421    p.set_defaults(func=bdev_ftl_get_stats)
2422
2423    def bdev_ftl_get_properties(args):
2424        print_dict(rpc.bdev.bdev_ftl_get_properties(args.client, name=args.name))
2425
2426    p = subparsers.add_parser('bdev_ftl_get_properties', help='Print FTL properties')
2427    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2428    p.set_defaults(func=bdev_ftl_get_properties)
2429
2430    def bdev_ftl_set_property(args):
2431        print_dict(rpc.bdev.bdev_ftl_set_property(args.client, name=args.name,
2432                   ftl_property=args.property,
2433                   value=args.value))
2434
2435    p = subparsers.add_parser('bdev_ftl_set_property', help='Set FTL property')
2436    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2437    p.add_argument('-p', '--property', help="Name of the property to be set", required=True)
2438    p.add_argument('-v', '--value', help="Value of the property", required=True)
2439    p.set_defaults(func=bdev_ftl_set_property)
2440
2441    # vmd
2442    def vmd_enable(args):
2443        print_dict(rpc.vmd.vmd_enable(args.client))
2444
2445    p = subparsers.add_parser('vmd_enable', aliases=['enable_vmd'], help='Enable VMD enumeration')
2446    p.set_defaults(func=vmd_enable)
2447
2448    def vmd_remove_device(args):
2449        print_dict(rpc.vmd.vmd_remove_device(args.client, addr=args.addr))
2450
2451    p = subparsers.add_parser('vmd_remove_device', help='Remove a device behind VMD')
2452    p.add_argument('addr', help='Address of the device to remove', type=str)
2453    p.set_defaults(func=vmd_remove_device)
2454
2455    def vmd_rescan(args):
2456        print_dict(rpc.vmd.vmd_rescan(args.client))
2457
2458    p = subparsers.add_parser('vmd_rescan', help='Force a rescan of the devices behind VMD')
2459    p.set_defaults(func=vmd_rescan)
2460
2461    # ublk
2462    def ublk_create_target(args):
2463        rpc.ublk.ublk_create_target(args.client,
2464                                    cpumask=args.cpumask,
2465                                    disable_user_copy=args.disable_user_copy)
2466    p = subparsers.add_parser('ublk_create_target',
2467                              help='Create spdk ublk target for ublk dev')
2468    p.add_argument('-m', '--cpumask', help='cpu mask for ublk dev')
2469    p.add_argument('--disable-user-copy', help='Disable user copy feature', action='store_true')
2470    p.set_defaults(func=ublk_create_target)
2471
2472    def ublk_destroy_target(args):
2473        rpc.ublk.ublk_destroy_target(args.client)
2474    p = subparsers.add_parser('ublk_destroy_target',
2475                              help='Destroy spdk ublk target for ublk dev')
2476    p.set_defaults(func=ublk_destroy_target)
2477
2478    def ublk_start_disk(args):
2479        print(rpc.ublk.ublk_start_disk(args.client,
2480                                       bdev_name=args.bdev_name,
2481                                       ublk_id=args.ublk_id,
2482                                       num_queues=args.num_queues,
2483                                       queue_depth=args.queue_depth))
2484
2485    p = subparsers.add_parser('ublk_start_disk',
2486                              help='Export a bdev as a ublk device')
2487    p.add_argument('bdev_name', help='Blockdev name to be exported. Example: Malloc0.')
2488    p.add_argument('ublk_id', help='ublk device id to be assigned. Example: 1.', type=int)
2489    p.add_argument('-q', '--num-queues', help="the total number of queues. Example: 1", type=int)
2490    p.add_argument('-d', '--queue-depth', help="queue depth. Example: 128", type=int)
2491    p.set_defaults(func=ublk_start_disk)
2492
2493    def ublk_stop_disk(args):
2494        rpc.ublk.ublk_stop_disk(args.client,
2495                                ublk_id=args.ublk_id)
2496
2497    p = subparsers.add_parser('ublk_stop_disk',
2498                              help='Stop a ublk device')
2499    p.add_argument('ublk_id', help='ublk device id to be deleted. Example: 1.', type=int)
2500    p.set_defaults(func=ublk_stop_disk)
2501
2502    def ublk_recover_disk(args):
2503        print(rpc.ublk.ublk_recover_disk(args.client,
2504                                         bdev_name=args.bdev_name,
2505                                         ublk_id=args.ublk_id))
2506
2507    p = subparsers.add_parser('ublk_recover_disk',
2508                              help='Recover ublk device')
2509    p.add_argument('bdev_name', help='Blockdev name to be recovered. Example: Malloc0.')
2510    p.add_argument('ublk_id', help='ublk device id to be recovered. Example: 1.', type=int)
2511    p.set_defaults(func=ublk_recover_disk)
2512
2513    def ublk_get_disks(args):
2514        print_dict(rpc.ublk.ublk_get_disks(args.client,
2515                                           ublk_id=args.ublk_id))
2516
2517    p = subparsers.add_parser('ublk_get_disks',
2518                              help='Display full or specified ublk device list')
2519    p.add_argument('-n', '--ublk-id', help="ublk device id. Example: 1", type=int)
2520    p.set_defaults(func=ublk_get_disks)
2521
2522    # nbd
2523    def nbd_start_disk(args):
2524        print(rpc.nbd.nbd_start_disk(args.client,
2525                                     bdev_name=args.bdev_name,
2526                                     nbd_device=args.nbd_device))
2527
2528    p = subparsers.add_parser('nbd_start_disk',
2529                              help='Export a bdev as an nbd disk')
2530    p.add_argument('bdev_name', help='Blockdev name to be exported. Example: Malloc0.')
2531    p.add_argument('nbd_device', help='Nbd device name to be assigned. Example: /dev/nbd0.', nargs='?')
2532    p.set_defaults(func=nbd_start_disk)
2533
2534    def nbd_stop_disk(args):
2535        rpc.nbd.nbd_stop_disk(args.client,
2536                              nbd_device=args.nbd_device)
2537
2538    p = subparsers.add_parser('nbd_stop_disk',
2539                              help='Stop an nbd disk')
2540    p.add_argument('nbd_device', help='Nbd device name to be stopped. Example: /dev/nbd0.')
2541    p.set_defaults(func=nbd_stop_disk)
2542
2543    def nbd_get_disks(args):
2544        print_dict(rpc.nbd.nbd_get_disks(args.client,
2545                                         nbd_device=args.nbd_device))
2546
2547    p = subparsers.add_parser('nbd_get_disks',
2548                              help='Display full or specified nbd device list')
2549    p.add_argument('-n', '--nbd-device', help="Path of the nbd device. Example: /dev/nbd0")
2550    p.set_defaults(func=nbd_get_disks)
2551
2552    # NVMe-oF
2553    def nvmf_set_max_subsystems(args):
2554        rpc.nvmf.nvmf_set_max_subsystems(args.client,
2555                                         max_subsystems=args.max_subsystems)
2556
2557    p = subparsers.add_parser('nvmf_set_max_subsystems',
2558                              help='Set the maximum number of NVMf target subsystems')
2559    p.add_argument('-x', '--max-subsystems', help='Max number of NVMf subsystems', type=int, required=True)
2560    p.set_defaults(func=nvmf_set_max_subsystems)
2561
2562    def nvmf_set_config(args):
2563        rpc.nvmf.nvmf_set_config(args.client,
2564                                 passthru_identify_ctrlr=args.passthru_identify_ctrlr,
2565                                 poll_groups_mask=args.poll_groups_mask,
2566                                 discovery_filter=args.discovery_filter,
2567                                 dhchap_digests=args.dhchap_digests,
2568                                 dhchap_dhgroups=args.dhchap_dhgroups)
2569
2570    p = subparsers.add_parser('nvmf_set_config', help='Set NVMf target config')
2571    p.add_argument('-i', '--passthru-identify-ctrlr', help="""Passthrough fields like serial number and model number
2572    when the controller has a single namespace that is an NVMe bdev""", action='store_true')
2573    p.add_argument('-m', '--poll-groups-mask', help='Set cpumask for NVMf poll groups (optional)', type=str)
2574    p.add_argument('-d', '--discovery-filter', help="""Set discovery filter (optional), possible values are: `match_any` (default) or
2575         comma separated values: `transport`, `address`, `svcid`""", type=str)
2576    p.add_argument('--dhchap-digests', help='Comma-separated list of allowed DH-HMAC-CHAP digests',
2577                   type=lambda d: d.split(','))
2578    p.add_argument('--dhchap-dhgroups', help='Comma-separated list of allowed DH-HMAC-CHAP DH groups',
2579                   type=lambda d: d.split(','))
2580    p.set_defaults(func=nvmf_set_config)
2581
2582    def nvmf_create_transport(args):
2583        rpc.nvmf.nvmf_create_transport(**vars(args))
2584
2585    p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport')
2586    p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True)
2587    p.add_argument('-g', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2588    p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int)
2589    p.add_argument('-m', '--max-io-qpairs-per-ctrlr', help='Max number of IO qpairs per controller', type=int)
2590    p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int)
2591    p.add_argument('-i', '--max-io-size', help='Max I/O size (bytes)', type=int)
2592    p.add_argument('-u', '--io-unit-size', help='I/O unit size (bytes)', type=int)
2593    p.add_argument('-a', '--max-aq-depth', help='Max number of admin cmds per AQ', type=int)
2594    p.add_argument('-n', '--num-shared-buffers', help='The number of pooled data buffers available to the transport', type=int)
2595    p.add_argument('-b', '--buf-cache-size', help='The number of shared buffers to reserve for each poll group', type=int)
2596    p.add_argument('-z', '--zcopy', action='store_true', help='''Use zero-copy operations if the
2597    underlying bdev supports them''')
2598    p.add_argument('-d', '--num-cqe', help="""The number of CQ entries. Only used when no_srq=true.
2599    Relevant only for RDMA transport""", type=int)
2600    p.add_argument('-s', '--max-srq-depth', help='Max number of outstanding I/O per SRQ. Relevant only for RDMA transport', type=int)
2601    p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport')
2602    p.add_argument('-o', '--c2h-success', action='store_false', help='Disable C2H success optimization. Relevant only for TCP transport')
2603    p.add_argument('-f', '--dif-insert-or-strip', action='store_true', help='Enable DIF insert/strip. Relevant only for TCP transport')
2604    p.add_argument('-y', '--sock-priority', help='The sock priority of the tcp connection. Relevant only for TCP transport', type=int)
2605    p.add_argument('-l', '--acceptor-backlog', help='Pending connections allowed at one time. Relevant only for RDMA transport', type=int)
2606    p.add_argument('-x', '--abort-timeout-sec', help='Abort execution timeout value, in seconds', type=int)
2607    p.add_argument('-w', '--no-wr-batching', action='store_true', help='Disable work requests batching. Relevant only for RDMA transport')
2608    p.add_argument('-e', '--control-msg-num', help="""The number of control messages per poll group.
2609    Relevant only for TCP transport""", type=int)
2610    p.add_argument('-M', '--disable-mappable-bar0', action='store_true', help="""Disable mmap() of BAR0.
2611    Relevant only for VFIO-USER transport""")
2612    p.add_argument('-I', '--disable-adaptive-irq', action='store_true', help="""Disable adaptive interrupt feature.
2613    Relevant only for VFIO-USER transport""")
2614    p.add_argument('-S', '--disable-shadow-doorbells', action='store_true', help="""Disable shadow doorbell support.
2615    Relevant only for VFIO-USER transport""")
2616    p.add_argument('--acceptor-poll-rate', help='Polling interval of the acceptor for incoming connections (usec)', type=int)
2617    p.add_argument('--ack-timeout', help='ACK timeout in milliseconds', type=int)
2618    p.add_argument('--data-wr-pool-size', help='RDMA data WR pool size. Relevant only for RDMA transport', type=int)
2619    p.add_argument('--disable-command-passthru', help='Disallow command passthru', action='store_true')
2620    p.set_defaults(func=nvmf_create_transport)
2621
2622    def nvmf_get_transports(args):
2623        print_dict(rpc.nvmf.nvmf_get_transports(args.client, trtype=args.trtype, tgt_name=args.tgt_name))
2624
2625    p = subparsers.add_parser('nvmf_get_transports', help='Display nvmf transports or required transport')
2626    p.add_argument('--trtype', help='Transport type (optional)')
2627    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2628    p.set_defaults(func=nvmf_get_transports)
2629
2630    def nvmf_get_subsystems(args):
2631        print_dict(rpc.nvmf.nvmf_get_subsystems(args.client, nqn=args.nqn, tgt_name=args.tgt_name))
2632
2633    p = subparsers.add_parser('nvmf_get_subsystems', help='Display nvmf subsystems or required subsystem')
2634    p.add_argument('nqn', help='Subsystem NQN (optional)', nargs="?")
2635    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2636    p.set_defaults(func=nvmf_get_subsystems)
2637
2638    def nvmf_create_subsystem(args):
2639        rpc.nvmf.nvmf_create_subsystem(args.client,
2640                                       nqn=args.nqn,
2641                                       tgt_name=args.tgt_name,
2642                                       serial_number=args.serial_number,
2643                                       model_number=args.model_number,
2644                                       allow_any_host=args.allow_any_host,
2645                                       max_namespaces=args.max_namespaces,
2646                                       ana_reporting=args.ana_reporting,
2647                                       min_cntlid=args.min_cntlid,
2648                                       max_cntlid=args.max_cntlid,
2649                                       max_discard_size_kib=args.max_discard_size,
2650                                       max_write_zeroes_size_kib=args.max_write_zeroes_size,
2651                                       passthrough=args.passthrough)
2652
2653    p = subparsers.add_parser('nvmf_create_subsystem', help='Create an NVMe-oF subsystem')
2654    p.add_argument('nqn', help='Subsystem NQN (ASCII)')
2655    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2656    p.add_argument("-s", "--serial-number", help="""
2657    Format:  'sn' etc
2658    Example: 'SPDK00000000000001'""")
2659    p.add_argument("-d", "--model-number", help="""
2660    Format:  'mn' etc
2661    Example: 'SPDK Controller'""")
2662    p.add_argument("-a", "--allow-any-host", action='store_true', help="Allow any host to connect (don't enforce allowed host NQN list)")
2663    p.add_argument("-m", "--max-namespaces", help="Maximum number of namespaces allowed",
2664                   type=int)
2665    p.add_argument("-r", "--ana-reporting", action='store_true', help="Enable ANA reporting feature")
2666    p.add_argument("-i", "--min_cntlid", help="Minimum controller ID", type=int)
2667    p.add_argument("-I", "--max_cntlid", help="Maximum controller ID", type=int)
2668    p.add_argument("--max-discard-size", help="Maximum discard size (Kib)", type=int)
2669    p.add_argument("--max-write-zeroes-size", help="Maximum write_zeroes size (Kib)", type=int)
2670    p.add_argument("-p", "--passthrough", action='store_true', help="""Use NVMe passthrough for all I/O commands and namespace-directed
2671                   admin commands""")
2672    p.set_defaults(func=nvmf_create_subsystem)
2673
2674    def nvmf_delete_subsystem(args):
2675        rpc.nvmf.nvmf_delete_subsystem(args.client,
2676                                       nqn=args.subsystem_nqn,
2677                                       tgt_name=args.tgt_name)
2678
2679    p = subparsers.add_parser('nvmf_delete_subsystem', help='Delete a nvmf subsystem')
2680    p.add_argument('subsystem_nqn',
2681                   help='subsystem nqn to be deleted. Example: nqn.2016-06.io.spdk:cnode1.')
2682    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2683    p.set_defaults(func=nvmf_delete_subsystem)
2684
2685    def nvmf_subsystem_add_listener(args):
2686        rpc.nvmf.nvmf_subsystem_add_listener(**vars(args))
2687
2688    p = subparsers.add_parser('nvmf_subsystem_add_listener', help='Add a listener to an NVMe-oF subsystem')
2689    p.add_argument('nqn', help='NVMe-oF subsystem NQN (\'discovery\' can be used as shortcut for discovery NQN)')
2690    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2691    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2692    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2693    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2694    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for RDMA or TCP)')
2695    p.add_argument('-k', '--secure-channel', help='Immediately establish a secure channel', action="store_true")
2696    p.add_argument('-n', '--ana-state', help='ANA state to set: optimized, non_optimized, or inaccessible', type=str)
2697    p.add_argument('-S', '--sock-impl', help='The socket implementation to use for the listener (ex. posix)', type=str)
2698    p.set_defaults(func=nvmf_subsystem_add_listener)
2699
2700    def nvmf_subsystem_remove_listener(args):
2701        rpc.nvmf.nvmf_subsystem_remove_listener(args.client,
2702                                                nqn=args.nqn,
2703                                                trtype=args.trtype,
2704                                                traddr=args.traddr,
2705                                                tgt_name=args.tgt_name,
2706                                                adrfam=args.adrfam,
2707                                                trsvcid=args.trsvcid)
2708
2709    p = subparsers.add_parser('nvmf_subsystem_remove_listener', help='Remove a listener from an NVMe-oF subsystem')
2710    p.add_argument('nqn', help='NVMe-oF subsystem NQN (\'discovery\' can be used as shortcut for discovery NQN)')
2711    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2712    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2713    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2714    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2715    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for TCP and RDMA transport types)')
2716    p.set_defaults(func=nvmf_subsystem_remove_listener)
2717
2718    def nvmf_subsystem_listener_set_ana_state(args):
2719        rpc.nvmf.nvmf_subsystem_listener_set_ana_state(args.client,
2720                                                       nqn=args.nqn,
2721                                                       ana_state=args.ana_state,
2722                                                       trtype=args.trtype,
2723                                                       traddr=args.traddr,
2724                                                       tgt_name=args.tgt_name,
2725                                                       adrfam=args.adrfam,
2726                                                       trsvcid=args.trsvcid,
2727                                                       anagrpid=args.anagrpid)
2728
2729    p = subparsers.add_parser('nvmf_subsystem_listener_set_ana_state', help='Set ANA state of a listener for an NVMe-oF subsystem')
2730    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2731    p.add_argument('-n', '--ana-state', help='ANA state to set: optimized, non_optimized, or inaccessible', required=True)
2732    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2733    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2734    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2735    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2736    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number')
2737    p.add_argument('-g', '--anagrpid', help='ANA group ID (optional)', type=int)
2738    p.set_defaults(func=nvmf_subsystem_listener_set_ana_state)
2739
2740    def nvmf_discovery_add_referral(args):
2741        rpc.nvmf.nvmf_discovery_add_referral(**vars(args))
2742
2743    p = subparsers.add_parser('nvmf_discovery_add_referral', help='Add a discovery service referral to an NVMe-oF target')
2744    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2745    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2746    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2747    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2748    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for RDMA or TCP)')
2749    p.add_argument('-k', '--secure-channel', help='The connection to that discovery subsystem requires a secure channel',
2750                   action="store_true")
2751    p.add_argument('-n', '--subnqn', help='Subsystem NQN')
2752    p.set_defaults(func=nvmf_discovery_add_referral)
2753
2754    def nvmf_discovery_remove_referral(args):
2755        rpc.nvmf.nvmf_discovery_remove_referral(args.client,
2756                                                trtype=args.trtype,
2757                                                traddr=args.traddr,
2758                                                tgt_name=args.tgt_name,
2759                                                adrfam=args.adrfam,
2760                                                trsvcid=args.trsvcid,
2761                                                subnqn=args.subnqn)
2762
2763    p = subparsers.add_parser('nvmf_discovery_remove_referral', help='Remove a discovery service referral from an NVMe-oF target')
2764    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2765    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2766    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2767    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2768    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for TCP and RDMA transport types)')
2769    p.add_argument('-n', '--subnqn', help='Subsystem NQN')
2770    p.set_defaults(func=nvmf_discovery_remove_referral)
2771
2772    def nvmf_discovery_get_referrals(args):
2773        print_dict(rpc.nvmf.nvmf_discovery_get_referrals(args.client,
2774                                                         tgt_name=args.tgt_name))
2775
2776    p = subparsers.add_parser('nvmf_discovery_get_referrals',
2777                              help='Display discovery subsystem referrals of an NVMe-oF target')
2778    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2779    p.set_defaults(func=nvmf_discovery_get_referrals)
2780
2781    def nvmf_subsystem_add_ns(args):
2782        rpc.nvmf.nvmf_subsystem_add_ns(**vars(args))
2783
2784    p = subparsers.add_parser('nvmf_subsystem_add_ns', help='Add a namespace to an NVMe-oF subsystem')
2785    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2786    p.add_argument('bdev_name', help='The name of the bdev that will back this namespace')
2787    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2788    p.add_argument('-p', '--ptpl-file', help='The persistent reservation storage location (optional)', type=str)
2789    p.add_argument('-n', '--nsid', help='The requested NSID (optional)', type=int)
2790    p.add_argument('-g', '--nguid', help='Namespace globally unique identifier (optional)')
2791    p.add_argument('-e', '--eui64', help='Namespace EUI-64 identifier (optional)')
2792    p.add_argument('-u', '--uuid', help='Namespace UUID (optional)')
2793    p.add_argument('-a', '--anagrpid', help='ANA group ID (optional)', type=int)
2794    p.add_argument('-i', '--no-auto-visible', action='store_true',
2795                   help='Do not auto make namespace visible to controllers (optional)')
2796    p.set_defaults(func=nvmf_subsystem_add_ns)
2797
2798    def nvmf_subsystem_remove_ns(args):
2799        rpc.nvmf.nvmf_subsystem_remove_ns(args.client,
2800                                          nqn=args.nqn,
2801                                          nsid=args.nsid,
2802                                          tgt_name=args.tgt_name)
2803
2804    p = subparsers.add_parser('nvmf_subsystem_remove_ns', help='Remove a namespace to an NVMe-oF subsystem')
2805    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2806    p.add_argument('nsid', help='The requested NSID', type=int)
2807    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2808    p.set_defaults(func=nvmf_subsystem_remove_ns)
2809
2810    def nvmf_ns_add_host(args):
2811        rpc.nvmf.nvmf_ns_visible(True,
2812                                 args.client,
2813                                 nqn=args.nqn,
2814                                 nsid=args.nsid,
2815                                 host=args.host,
2816                                 tgt_name=args.tgt_name)
2817
2818    def nvmf_ns_visible_add_args(p):
2819        p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2820        p.add_argument('nsid', help='The requested NSID', type=int)
2821        p.add_argument('host', help='Host NQN to make namespace visible to')
2822        p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2823
2824    p = subparsers.add_parser('nvmf_ns_add_host', help='Make namespace visible to controllers of host')
2825    nvmf_ns_visible_add_args(p)
2826    p.set_defaults(func=nvmf_ns_add_host)
2827
2828    def nvmf_ns_remove_host(args):
2829        rpc.nvmf.nvmf_ns_visible(False,
2830                                 args.client,
2831                                 nqn=args.nqn,
2832                                 nsid=args.nsid,
2833                                 host=args.host,
2834                                 tgt_name=args.tgt_name)
2835
2836    p = subparsers.add_parser('nvmf_ns_remove_host', help='Make namespace not visible to controllers of host')
2837    nvmf_ns_visible_add_args(p)
2838    p.set_defaults(func=nvmf_ns_remove_host)
2839
2840    def nvmf_subsystem_add_host(args):
2841        rpc.nvmf.nvmf_subsystem_add_host(args.client,
2842                                         nqn=args.nqn,
2843                                         host=args.host,
2844                                         tgt_name=args.tgt_name,
2845                                         psk=args.psk,
2846                                         dhchap_key=args.dhchap_key,
2847                                         dhchap_ctrlr_key=args.dhchap_ctrlr_key)
2848
2849    p = subparsers.add_parser('nvmf_subsystem_add_host', help='Add a host to an NVMe-oF subsystem')
2850    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2851    p.add_argument('host', help='Host NQN to allow')
2852    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2853    p.add_argument('--psk', help='Path to PSK file for TLS authentication (optional). Only applicable for TCP transport.', type=str)
2854    p.add_argument('--dhchap-key', help='DH-HMAC-CHAP key name (optional)')
2855    p.add_argument('--dhchap-ctrlr-key', help='DH-HMAC-CHAP controller key name (optional)')
2856    p.set_defaults(func=nvmf_subsystem_add_host)
2857
2858    def nvmf_subsystem_remove_host(args):
2859        rpc.nvmf.nvmf_subsystem_remove_host(args.client,
2860                                            nqn=args.nqn,
2861                                            host=args.host,
2862                                            tgt_name=args.tgt_name)
2863
2864    p = subparsers.add_parser('nvmf_subsystem_remove_host', help='Remove a host from an NVMe-oF subsystem')
2865    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2866    p.add_argument('host', help='Host NQN to remove')
2867    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2868    p.set_defaults(func=nvmf_subsystem_remove_host)
2869
2870    def nvmf_subsystem_allow_any_host(args):
2871        rpc.nvmf.nvmf_subsystem_allow_any_host(args.client,
2872                                               nqn=args.nqn,
2873                                               disable=args.disable,
2874                                               tgt_name=args.tgt_name)
2875
2876    p = subparsers.add_parser('nvmf_subsystem_allow_any_host', help='Allow any host to connect to the subsystem')
2877    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2878    p.add_argument('-e', '--enable', action='store_true', help='Enable allowing any host')
2879    p.add_argument('-d', '--disable', action='store_true', help='Disable allowing any host')
2880    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2881    p.set_defaults(func=nvmf_subsystem_allow_any_host)
2882
2883    def nvmf_subsystem_get_controllers(args):
2884        print_dict(rpc.nvmf.nvmf_subsystem_get_controllers(args.client,
2885                                                           nqn=args.nqn,
2886                                                           tgt_name=args.tgt_name))
2887
2888    p = subparsers.add_parser('nvmf_subsystem_get_controllers',
2889                              help='Display controllers of an NVMe-oF subsystem.')
2890    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2891    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2892    p.set_defaults(func=nvmf_subsystem_get_controllers)
2893
2894    def nvmf_subsystem_get_qpairs(args):
2895        print_dict(rpc.nvmf.nvmf_subsystem_get_qpairs(args.client,
2896                                                      nqn=args.nqn,
2897                                                      tgt_name=args.tgt_name))
2898
2899    p = subparsers.add_parser('nvmf_subsystem_get_qpairs',
2900                              help='Display queue pairs of an NVMe-oF subsystem.')
2901    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2902    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2903    p.set_defaults(func=nvmf_subsystem_get_qpairs)
2904
2905    def nvmf_subsystem_get_listeners(args):
2906        print_dict(rpc.nvmf.nvmf_subsystem_get_listeners(args.client,
2907                                                         nqn=args.nqn,
2908                                                         tgt_name=args.tgt_name))
2909
2910    p = subparsers.add_parser('nvmf_subsystem_get_listeners',
2911                              help='Display listeners of an NVMe-oF subsystem.')
2912    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2913    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2914    p.set_defaults(func=nvmf_subsystem_get_listeners)
2915
2916    def nvmf_get_stats(args):
2917        print_dict(rpc.nvmf.nvmf_get_stats(args.client, tgt_name=args.tgt_name))
2918
2919    p = subparsers.add_parser(
2920        'nvmf_get_stats', help='Display current statistics for NVMf subsystem')
2921    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2922    p.set_defaults(func=nvmf_get_stats)
2923
2924    def nvmf_set_crdt(args):
2925        print_dict(rpc.nvmf.nvmf_set_crdt(args.client, args.crdt1, args.crdt2, args.crdt3))
2926
2927    p = subparsers.add_parser(
2928        'nvmf_set_crdt',
2929        help="""Set the 3 crdt (Command Retry Delay Time) values for NVMf subsystem. All
2930        values are in units of 100 milliseconds (same as the NVM Express specification).""")
2931    p.add_argument('-t1', '--crdt1', help='Command Retry Delay Time 1, in units of 100 milliseconds', type=int)
2932    p.add_argument('-t2', '--crdt2', help='Command Retry Delay Time 2, in units of 100 milliseconds', type=int)
2933    p.add_argument('-t3', '--crdt3', help='Command Retry Delay Time 3, in units of 100 milliseconds', type=int)
2934    p.set_defaults(func=nvmf_set_crdt)
2935
2936    def nvmf_publish_mdns_prr(args):
2937        rpc.nvmf.nvmf_publish_mdns_prr(args.client, args.tgt_name)
2938
2939    p = subparsers.add_parser('nvmf_publish_mdns_prr',
2940                              help='Publish pull registration request through mdns')
2941    p.add_argument('-t', '--tgt-name', help='The name of the NVMe-oF target (optional)', type=str)
2942    p.set_defaults(func=nvmf_publish_mdns_prr)
2943
2944    def nvmf_stop_mdns_prr(args):
2945        rpc.nvmf.nvmf_stop_mdns_prr(args.client, args.tgt_name)
2946
2947    p = subparsers.add_parser('nvmf_stop_mdns_prr',
2948                              help='Stop publishing pull registration request through mdns')
2949    p.add_argument('-t', '--tgt-name', help='The name of the NVMe-oF target (optional)', type=str)
2950    p.set_defaults(func=nvmf_stop_mdns_prr)
2951
2952    # subsystem
2953    def framework_get_subsystems(args):
2954        print_dict(rpc.subsystem.framework_get_subsystems(args.client))
2955
2956    p = subparsers.add_parser('framework_get_subsystems',
2957                              help="""Print subsystems array in initialization order. Each subsystem
2958    entry contain (unsorted) array of subsystems it depends on.""")
2959    p.set_defaults(func=framework_get_subsystems)
2960
2961    def framework_get_config(args):
2962        print_dict(rpc.subsystem.framework_get_config(args.client, args.name))
2963
2964    p = subparsers.add_parser('framework_get_config', help="""Print subsystem configuration""")
2965    p.add_argument('name', help='Name of subsystem to query')
2966    p.set_defaults(func=framework_get_config)
2967
2968    # vhost
2969    def vhost_controller_set_coalescing(args):
2970        rpc.vhost.vhost_controller_set_coalescing(args.client,
2971                                                  ctrlr=args.ctrlr,
2972                                                  delay_base_us=args.delay_base_us,
2973                                                  iops_threshold=args.iops_threshold)
2974
2975    p = subparsers.add_parser('vhost_controller_set_coalescing', help='Set vhost controller coalescing')
2976    p.add_argument('ctrlr', help='controller name')
2977    p.add_argument('delay_base_us', help='Base delay time', type=int)
2978    p.add_argument('iops_threshold', help='IOPS threshold when coalescing is enabled', type=int)
2979    p.set_defaults(func=vhost_controller_set_coalescing)
2980
2981    def virtio_blk_create_transport(args):
2982        rpc.vhost.virtio_blk_create_transport(**vars(args))
2983
2984    p = subparsers.add_parser('virtio_blk_create_transport',
2985                              help='Create virtio blk transport')
2986    p.add_argument('name', help='transport name')
2987    p.set_defaults(func=virtio_blk_create_transport)
2988
2989    def virtio_blk_get_transports(args):
2990        print_dict(rpc.vhost.virtio_blk_get_transports(args.client, name=args.name))
2991
2992    p = subparsers.add_parser('virtio_blk_get_transports', help='Display virtio-blk transports or requested transport')
2993    p.add_argument('--name', help='Transport name (optional)', type=str)
2994    p.set_defaults(func=virtio_blk_get_transports)
2995
2996    def vhost_create_scsi_controller(args):
2997        rpc.vhost.vhost_create_scsi_controller(args.client,
2998                                               ctrlr=args.ctrlr,
2999                                               cpumask=args.cpumask,
3000                                               delay=args.delay)
3001
3002    p = subparsers.add_parser('vhost_create_scsi_controller', help='Add new vhost controller')
3003    p.add_argument('ctrlr', help='controller name')
3004    p.add_argument('--cpumask', help='cpu mask for this controller')
3005    p.add_argument("--delay", action='store_true', help='whether delay starting controller or not')
3006    p.set_defaults(func=vhost_create_scsi_controller)
3007
3008    def vhost_start_scsi_controller(args):
3009        rpc.vhost.vhost_start_scsi_controller(args.client,
3010                                              ctrlr=args.ctrlr)
3011
3012    p = subparsers.add_parser('vhost_start_scsi_controller', help='Start vhost scsi controller')
3013    p.add_argument('ctrlr', help='controller name')
3014    p.set_defaults(func=vhost_start_scsi_controller)
3015
3016    def vhost_scsi_controller_add_target(args):
3017        print_json(rpc.vhost.vhost_scsi_controller_add_target(args.client,
3018                                                              ctrlr=args.ctrlr,
3019                                                              scsi_target_num=args.scsi_target_num,
3020                                                              bdev_name=args.bdev_name))
3021
3022    p = subparsers.add_parser('vhost_scsi_controller_add_target', help='Add lun to vhost controller')
3023    p.add_argument('ctrlr', help='controller name where add lun')
3024    p.add_argument('scsi_target_num', help='scsi_target_num', type=int)
3025    p.add_argument('bdev_name', help='bdev name')
3026    p.set_defaults(func=vhost_scsi_controller_add_target)
3027
3028    def vhost_scsi_controller_remove_target(args):
3029        rpc.vhost.vhost_scsi_controller_remove_target(args.client,
3030                                                      ctrlr=args.ctrlr,
3031                                                      scsi_target_num=args.scsi_target_num)
3032
3033    p = subparsers.add_parser('vhost_scsi_controller_remove_target',
3034                              help='Remove target from vhost controller')
3035    p.add_argument('ctrlr', help='controller name to remove target from')
3036    p.add_argument('scsi_target_num', help='scsi_target_num', type=int)
3037    p.set_defaults(func=vhost_scsi_controller_remove_target)
3038
3039    def vhost_create_blk_controller(args):
3040        rpc.vhost.vhost_create_blk_controller(**vars(args))
3041
3042    p = subparsers.add_parser('vhost_create_blk_controller', help='Add a new vhost block controller')
3043    p.add_argument('ctrlr', help='controller name')
3044    p.add_argument('dev_name', help='device name')
3045    p.add_argument('--cpumask', help='cpu mask for this controller')
3046    p.add_argument('--transport', help='virtio blk transport name (default: vhost_user_blk)')
3047    p.add_argument("-r", "--readonly", action='store_true', help='Set controller as read-only')
3048    p.add_argument("-p", "--packed_ring", action='store_true', help='Set controller as packed ring supported')
3049    p.set_defaults(func=vhost_create_blk_controller)
3050
3051    def vhost_get_controllers(args):
3052        print_dict(rpc.vhost.vhost_get_controllers(args.client, args.name))
3053
3054    p = subparsers.add_parser('vhost_get_controllers', help='List all or specific vhost controller(s)')
3055    p.add_argument('-n', '--name', help="Name of vhost controller")
3056    p.set_defaults(func=vhost_get_controllers)
3057
3058    def vhost_delete_controller(args):
3059        rpc.vhost.vhost_delete_controller(args.client,
3060                                          ctrlr=args.ctrlr)
3061
3062    p = subparsers.add_parser('vhost_delete_controller', help='Delete a vhost controller')
3063    p.add_argument('ctrlr', help='controller name')
3064    p.set_defaults(func=vhost_delete_controller)
3065
3066    def bdev_virtio_attach_controller(args):
3067        print_array(rpc.vhost.bdev_virtio_attach_controller(args.client,
3068                                                            name=args.name,
3069                                                            trtype=args.trtype,
3070                                                            traddr=args.traddr,
3071                                                            dev_type=args.dev_type,
3072                                                            vq_count=args.vq_count,
3073                                                            vq_size=args.vq_size))
3074
3075    p = subparsers.add_parser('bdev_virtio_attach_controller',
3076                              help="""Attach virtio controller using provided
3077    transport type and device type. This will also create bdevs for any block devices connected to the
3078    controller (for example, SCSI devices for a virtio-scsi controller).
3079    Result is array of added bdevs.""")
3080    p.add_argument('name', help="Use this name as base for new created bdevs")
3081    p.add_argument('-t', '--trtype',
3082                   help='Virtio target transport type: pci or user', required=True)
3083    p.add_argument('-a', '--traddr',
3084                   help='Transport type specific target address: e.g. UNIX domain socket path or BDF', required=True)
3085    p.add_argument('-d', '--dev-type',
3086                   help='Device type: blk or scsi', required=True)
3087    p.add_argument('--vq-count', help='Number of virtual queues to be used.', type=int)
3088    p.add_argument('--vq-size', help='Size of each queue', type=int)
3089    p.set_defaults(func=bdev_virtio_attach_controller)
3090
3091    def bdev_virtio_scsi_get_devices(args):
3092        print_dict(rpc.vhost.bdev_virtio_scsi_get_devices(args.client))
3093
3094    p = subparsers.add_parser('bdev_virtio_scsi_get_devices', help='List all Virtio-SCSI devices.')
3095    p.set_defaults(func=bdev_virtio_scsi_get_devices)
3096
3097    def bdev_virtio_detach_controller(args):
3098        rpc.vhost.bdev_virtio_detach_controller(args.client,
3099                                                name=args.name)
3100
3101    p = subparsers.add_parser('bdev_virtio_detach_controller', help="""Remove a Virtio device
3102    This will delete all bdevs exposed by this device""")
3103    p.add_argument('name', help='Virtio device name. E.g. VirtioUser0')
3104    p.set_defaults(func=bdev_virtio_detach_controller)
3105
3106    def bdev_virtio_blk_set_hotplug(args):
3107        rpc.vhost.bdev_virtio_blk_set_hotplug(args.client, enable=args.enable, period_us=args.period_us)
3108
3109    p = subparsers.add_parser('bdev_virtio_blk_set_hotplug', help='Set hotplug options for bdev virtio_blk type.')
3110    p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)")
3111    p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug")
3112    p.add_argument('-r', '--period-us',
3113                   help='How often the hotplug is processed for insert and remove events', type=int)
3114    p.set_defaults(func=bdev_virtio_blk_set_hotplug)
3115
3116    # vfio-user target
3117    def vfu_tgt_set_base_path(args):
3118        rpc.vfio_user.vfu_tgt_set_base_path(args.client, path=args.path)
3119
3120    p = subparsers.add_parser('vfu_tgt_set_base_path', help='Set socket base path.')
3121    p.add_argument('path', help='socket base path')
3122    p.set_defaults(func=vfu_tgt_set_base_path)
3123
3124    def vfu_virtio_delete_endpoint(args):
3125        rpc.vfio_user.vfu_virtio_delete_endpoint(args.client, name=args.name)
3126
3127    p = subparsers.add_parser('vfu_virtio_delete_endpoint', help='Delete the PCI device via endpoint name.')
3128    p.add_argument('name', help='Endpoint name')
3129    p.set_defaults(func=vfu_virtio_delete_endpoint)
3130
3131    def vfu_virtio_create_blk_endpoint(args):
3132        rpc.vfio_user.vfu_virtio_create_blk_endpoint(args.client,
3133                                                     name=args.name,
3134                                                     bdev_name=args.bdev_name,
3135                                                     cpumask=args.cpumask,
3136                                                     num_queues=args.num_queues,
3137                                                     qsize=args.qsize,
3138                                                     packed_ring=args.packed_ring)
3139
3140    p = subparsers.add_parser('vfu_virtio_create_blk_endpoint', help='Create virtio-blk endpoint.')
3141    p.add_argument('name', help='Name of the endpoint')
3142    p.add_argument('--bdev-name', help='block device name', type=str, required=True)
3143    p.add_argument('--cpumask', help='CPU masks')
3144    p.add_argument('--num-queues', help='number of vrings', type=int, default=0)
3145    p.add_argument('--qsize', help='number of element for each vring', type=int, default=0)
3146    p.add_argument("--packed-ring", action='store_true', help='Enable packed ring')
3147    p.set_defaults(func=vfu_virtio_create_blk_endpoint)
3148
3149    def vfu_virtio_scsi_add_target(args):
3150        rpc.vfio_user.vfu_virtio_scsi_add_target(args.client,
3151                                                 name=args.name,
3152                                                 scsi_target_num=args.scsi_target_num,
3153                                                 bdev_name=args.bdev_name)
3154
3155    p = subparsers.add_parser('vfu_virtio_scsi_add_target', help='Attach a block device to SCSI target of PCI endpoint.')
3156    p.add_argument('name', help='Name of the endpoint')
3157    p.add_argument('--scsi-target-num', help='number of SCSI Target', type=int, required=True)
3158    p.add_argument('--bdev-name', help='block device name', type=str, required=True)
3159    p.set_defaults(func=vfu_virtio_scsi_add_target)
3160
3161    def vfu_virtio_scsi_remove_target(args):
3162        rpc.vfio_user.vfu_virtio_scsi_remove_target(args.client,
3163                                                    name=args.name,
3164                                                    scsi_target_num=args.scsi_target_num)
3165
3166    p = subparsers.add_parser('vfu_virtio_scsi_remove_target', help='Remove the specified SCSI target of PCI endpoint.')
3167    p.add_argument('name', help='Name of the endpoint')
3168    p.add_argument('--scsi-target-num', help='number of SCSI Target', type=int, required=True)
3169    p.set_defaults(func=vfu_virtio_scsi_remove_target)
3170
3171    def vfu_virtio_create_scsi_endpoint(args):
3172        rpc.vfio_user.vfu_virtio_create_scsi_endpoint(args.client,
3173                                                      name=args.name,
3174                                                      cpumask=args.cpumask,
3175                                                      num_io_queues=args.num_io_queues,
3176                                                      qsize=args.qsize,
3177                                                      packed_ring=args.packed_ring)
3178
3179    p = subparsers.add_parser('vfu_virtio_create_scsi_endpoint', help='Create virtio-scsi endpoint.')
3180    p.add_argument('name', help='Name of the endpoint')
3181    p.add_argument('--cpumask', help='CPU masks')
3182    p.add_argument('--num-io-queues', help='number of IO vrings', type=int, default=0)
3183    p.add_argument('--qsize', help='number of element for each vring', type=int, default=0)
3184    p.add_argument("--packed-ring", action='store_true', help='Enable packed ring')
3185    p.set_defaults(func=vfu_virtio_create_scsi_endpoint)
3186
3187    def vfu_virtio_create_fs_endpoint(args):
3188        rpc.vfio_user.vfu_virtio_create_fs_endpoint(args.client,
3189                                                    name=args.name,
3190                                                    fsdev_name=args.fsdev_name,
3191                                                    tag=args.tag,
3192                                                    cpumask=args.cpumask,
3193                                                    num_queues=args.num_queues,
3194                                                    qsize=args.qsize,
3195                                                    packed_ring=args.packed_ring)
3196
3197    p = subparsers.add_parser('vfu_virtio_create_fs_endpoint', help='Create virtio-fs endpoint.')
3198    p.add_argument('name', help='Name of the endpoint')
3199    p.add_argument('--fsdev-name', help='fsdev name', type=str, required=True)
3200    p.add_argument('--tag', help='virtiofs tag', type=str, required=True)
3201    p.add_argument('--cpumask', help='CPU masks')
3202    p.add_argument('--num-queues', help='number of vrings', type=int, default=0)
3203    p.add_argument('--qsize', help='number of element for each vring', type=int, default=0)
3204    p.add_argument("--packed-ring", action='store_true', help='Enable packed ring')
3205    p.set_defaults(func=vfu_virtio_create_fs_endpoint)
3206
3207    # accel_fw
3208    def accel_get_opc_assignments(args):
3209        print_dict(rpc.accel.accel_get_opc_assignments(args.client))
3210
3211    p = subparsers.add_parser('accel_get_opc_assignments', help='Get list of opcode name to module assignments.')
3212    p.set_defaults(func=accel_get_opc_assignments)
3213
3214    def accel_get_module_info(args):
3215        print_dict(rpc.accel.accel_get_module_info(args.client))
3216
3217    p = subparsers.add_parser('accel_get_module_info', aliases=['accel_get_engine_info'],
3218                              help='Get list of valid module names and their operations.')
3219    p.set_defaults(func=accel_get_module_info)
3220
3221    def accel_assign_opc(args):
3222        rpc.accel.accel_assign_opc(args.client, opname=args.opname, module=args.module)
3223
3224    p = subparsers.add_parser('accel_assign_opc', help='Manually assign an operation to a module.')
3225    p.add_argument('-o', '--opname', help='opname')
3226    p.add_argument('-m', '--module', help='name of module')
3227    p.set_defaults(func=accel_assign_opc)
3228
3229    def accel_crypto_key_create(args):
3230        print_dict(rpc.accel.accel_crypto_key_create(args.client,
3231                                                     cipher=args.cipher,
3232                                                     key=args.key,
3233                                                     key2=args.key2,
3234                                                     tweak_mode=args.tweak_mode,
3235                                                     name=args.name))
3236
3237    p = subparsers.add_parser('accel_crypto_key_create', help='Create encryption key')
3238    p.add_argument('-c', '--cipher', help='cipher', required=True)
3239    p.add_argument('-k', '--key', help='key', required=True)
3240    p.add_argument('-e', '--key2', help='key2', default=None)
3241    p.add_argument('-t', '--tweak-mode', help='tweak mode', default=None)
3242    p.add_argument('-n', '--name', help='key name', required=True)
3243    p.set_defaults(func=accel_crypto_key_create)
3244
3245    def accel_crypto_key_destroy(args):
3246        print_dict(rpc.accel.accel_crypto_key_destroy(args.client,
3247                                                      key_name=args.name))
3248
3249    p = subparsers.add_parser('accel_crypto_key_destroy', help='Destroy encryption key')
3250    p.add_argument('-n', '--name', help='key name', required=True, type=str)
3251    p.set_defaults(func=accel_crypto_key_destroy)
3252
3253    def accel_crypto_keys_get(args):
3254        print_dict(rpc.accel.accel_crypto_keys_get(args.client,
3255                                                   key_name=args.key_name))
3256
3257    p = subparsers.add_parser('accel_crypto_keys_get', help='Get a list of the crypto keys')
3258    p.add_argument('-k', '--key-name', help='Get information about a specific key', type=str)
3259    p.set_defaults(func=accel_crypto_keys_get)
3260
3261    def accel_set_driver(args):
3262        rpc.accel.accel_set_driver(args.client, name=args.name)
3263
3264    p = subparsers.add_parser('accel_set_driver', help='Select accel platform driver to execute ' +
3265                              'operation chains')
3266    p.add_argument('name', help='name of the platform driver')
3267    p.set_defaults(func=accel_set_driver)
3268
3269    def accel_set_options(args):
3270        rpc.accel.accel_set_options(args.client, args.small_cache_size, args.large_cache_size,
3271                                    args.task_count, args.sequence_count, args.buf_count)
3272
3273    p = subparsers.add_parser('accel_set_options', help='Set accel framework\'s options')
3274    p.add_argument('--small-cache-size', type=int, help='Size of the small iobuf cache')
3275    p.add_argument('--large-cache-size', type=int, help='Size of the large iobuf cache')
3276    p.add_argument('--task-count', type=int, help='Maximum number of tasks per IO channel')
3277    p.add_argument('--sequence-count', type=int, help='Maximum number of sequences per IO channel')
3278    p.add_argument('--buf-count', type=int, help='Maximum number of buffers per IO channel')
3279    p.set_defaults(func=accel_set_options)
3280
3281    def accel_get_stats(args):
3282        print_dict(rpc.accel.accel_get_stats(args.client))
3283
3284    p = subparsers.add_parser('accel_get_stats', help='Display accel framework\'s statistics')
3285    p.set_defaults(func=accel_get_stats)
3286
3287    # ioat
3288    def ioat_scan_accel_module(args):
3289        rpc.ioat.ioat_scan_accel_module(args.client)
3290
3291    p = subparsers.add_parser('ioat_scan_accel_module', aliases=['ioat_scan_accel_engine'],
3292                              help='Enable IOAT accel module offload.')
3293    p.set_defaults(func=ioat_scan_accel_module)
3294
3295    # dpdk compressdev
3296    def compressdev_scan_accel_module(args):
3297        rpc.compressdev.compressdev_scan_accel_module(args.client, pmd=args.pmd)
3298
3299    p = subparsers.add_parser('compressdev_scan_accel_module', help='Scan and enable compressdev module and set pmd option.')
3300    p.add_argument('-p', '--pmd', type=int, help='0 = auto-select, 1= QAT only, 2 = mlx5_pci only')
3301    p.set_defaults(func=compressdev_scan_accel_module)
3302
3303    # dsa
3304    def dsa_scan_accel_module(args):
3305        rpc.dsa.dsa_scan_accel_module(args.client, config_kernel_mode=args.config_kernel_mode)
3306
3307    p = subparsers.add_parser('dsa_scan_accel_module', aliases=['dsa_scan_accel_engine'],
3308                              help='Set config and enable dsa accel module offload.')
3309    p.add_argument('-k', '--config-kernel-mode', help='Use Kernel mode dsa',
3310                   action='store_true', dest='config_kernel_mode')
3311    p.set_defaults(func=dsa_scan_accel_module, config_kernel_mode=None)
3312
3313    # iaa
3314    def iaa_scan_accel_module(args):
3315        rpc.iaa.iaa_scan_accel_module(args.client)
3316
3317    p = subparsers.add_parser('iaa_scan_accel_module', aliases=['iaa_scan_accel_engine'],
3318                              help='Set config and enable iaa accel module offload.')
3319    p.set_defaults(func=iaa_scan_accel_module)
3320
3321    def dpdk_cryptodev_scan_accel_module(args):
3322        rpc.dpdk_cryptodev.dpdk_cryptodev_scan_accel_module(args.client)
3323
3324    p = subparsers.add_parser('dpdk_cryptodev_scan_accel_module',
3325                              help='Enable dpdk_cryptodev accel module offload.')
3326    p.set_defaults(func=dpdk_cryptodev_scan_accel_module)
3327
3328    def dpdk_cryptodev_set_driver(args):
3329        rpc.dpdk_cryptodev.dpdk_cryptodev_set_driver(args.client,
3330                                                     driver_name=args.driver_name)
3331
3332    p = subparsers.add_parser('dpdk_cryptodev_set_driver',
3333                              help='Set the DPDK cryptodev driver.')
3334    p.add_argument('-d', '--driver-name', help='The driver, can be one of crypto_aesni_mb, crypto_qat or mlx5_pci', type=str)
3335    p.set_defaults(func=dpdk_cryptodev_set_driver)
3336
3337    def dpdk_cryptodev_get_driver(args):
3338        print_dict(rpc.dpdk_cryptodev.dpdk_cryptodev_get_driver(args.client))
3339
3340    p = subparsers.add_parser('dpdk_cryptodev_get_driver', help='Get the DPDK cryptodev driver')
3341    p.set_defaults(func=dpdk_cryptodev_get_driver)
3342
3343    # mlx5
3344    def mlx5_scan_accel_module(args):
3345        rpc.mlx5.mlx5_scan_accel_module(args.client,
3346                                        qp_size=args.qp_size,
3347                                        num_requests=args.num_requests,
3348                                        allowed_devs=args.allowed_devs,
3349                                        crypto_split_blocks=args.crypto_split_blocks)
3350
3351    p = subparsers.add_parser('mlx5_scan_accel_module', help='Enable mlx5 accel module.')
3352    p.add_argument('-q', '--qp-size', type=int, help='QP size')
3353    p.add_argument('-r', '--num-requests', type=int, help='Size of the shared requests pool')
3354    p.add_argument('-d', '--allowed-devs', help="Comma separated list of allowed device names, e.g. mlx5_0,mlx5_1")
3355    p.add_argument('-s', '--crypto-split-blocks', type=int,
3356                   help="Number of data blocks to be processed in 1 crypto UMR. [0-65535], 0 means no limit")
3357    p.set_defaults(func=mlx5_scan_accel_module)
3358
3359    # accel_error
3360    def accel_error_inject_error(args):
3361        rpc.accel.accel_error_inject_error(args.client, opcode=args.opcode,
3362                                           type=args.type, count=args.count,
3363                                           interval=args.interval, errcode=args.errcode)
3364
3365    p = subparsers.add_parser('accel_error_inject_error',
3366                              help='Inject an error to processing accel operation')
3367    p.add_argument('-o', '--opcode', help='Opcode')
3368    p.add_argument('-t', '--type',
3369                   help='Error type ("corrupt": corrupt the data, "failure": fail the operation, "disable": disable error injection)')
3370    p.add_argument('-c', '--count', type=int,
3371                   help='Number of errors to inject on each IO channel (0 to disable error injection)')
3372    p.add_argument('-i', '--interval', type=int, help='Interval between injections')
3373    p.add_argument('--errcode', type=int, help='Error code to inject (only relevant for type=failure)')
3374    p.set_defaults(func=accel_error_inject_error)
3375
3376    # opal
3377    def bdev_nvme_opal_init(args):
3378        rpc.nvme.bdev_nvme_opal_init(args.client,
3379                                     nvme_ctrlr_name=args.nvme_ctrlr_name,
3380                                     password=args.password)
3381
3382    p = subparsers.add_parser('bdev_nvme_opal_init', help='take ownership and activate')
3383    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name')
3384    p.add_argument('-p', '--password', help='password for admin')
3385    p.set_defaults(func=bdev_nvme_opal_init)
3386
3387    def bdev_nvme_opal_revert(args):
3388        rpc.nvme.bdev_nvme_opal_revert(args.client,
3389                                       nvme_ctrlr_name=args.nvme_ctrlr_name,
3390                                       password=args.password)
3391    p = subparsers.add_parser('bdev_nvme_opal_revert', help='Revert to default factory settings')
3392    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name')
3393    p.add_argument('-p', '--password', help='password')
3394    p.set_defaults(func=bdev_nvme_opal_revert)
3395
3396    def bdev_opal_create(args):
3397        print_json(rpc.bdev.bdev_opal_create(args.client,
3398                                             nvme_ctrlr_name=args.nvme_ctrlr_name,
3399                                             nsid=args.nsid,
3400                                             locking_range_id=args.locking_range_id,
3401                                             range_start=args.range_start,
3402                                             range_length=args.range_length,
3403                                             password=args.password))
3404
3405    p = subparsers.add_parser('bdev_opal_create', help="""Create opal bdev on specified NVMe controller""")
3406    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name', required=True)
3407    p.add_argument('-n', '--nsid', help='namespace ID (only support nsid=1 for now)', type=int, required=True)
3408    p.add_argument('-i', '--locking-range-id', help='locking range id', type=int, required=True)
3409    p.add_argument('-s', '--range-start', help='locking range start LBA', type=int, required=True)
3410    p.add_argument('-l', '--range-length', help='locking range length (in blocks)', type=int, required=True)
3411    p.add_argument('-p', '--password', help='admin password', required=True)
3412    p.set_defaults(func=bdev_opal_create)
3413
3414    def bdev_opal_get_info(args):
3415        print_dict(rpc.bdev.bdev_opal_get_info(args.client,
3416                                               bdev_name=args.bdev_name,
3417                                               password=args.password))
3418
3419    p = subparsers.add_parser('bdev_opal_get_info', help='get opal locking range info for this bdev')
3420    p.add_argument('-b', '--bdev-name', help='opal bdev')
3421    p.add_argument('-p', '--password', help='password')
3422    p.set_defaults(func=bdev_opal_get_info)
3423
3424    def bdev_opal_delete(args):
3425        rpc.bdev.bdev_opal_delete(args.client,
3426                                  bdev_name=args.bdev_name,
3427                                  password=args.password)
3428
3429    p = subparsers.add_parser('bdev_opal_delete', help="""delete a virtual opal bdev""")
3430    p.add_argument('-b', '--bdev-name', help='opal virtual bdev', required=True)
3431    p.add_argument('-p', '--password', help='admin password', required=True)
3432    p.set_defaults(func=bdev_opal_delete)
3433
3434    def bdev_opal_new_user(args):
3435        rpc.bdev.bdev_opal_new_user(args.client,
3436                                    bdev_name=args.bdev_name,
3437                                    admin_password=args.admin_password,
3438                                    user_id=args.user_id,
3439                                    user_password=args.user_password)
3440
3441    p = subparsers.add_parser('bdev_opal_new_user', help="""Add a user to opal bdev who can set lock state for this bdev""")
3442    p.add_argument('-b', '--bdev-name', help='opal bdev', required=True)
3443    p.add_argument('-p', '--admin-password', help='admin password', required=True)
3444    p.add_argument('-i', '--user-id', help='ID for new user', type=int, required=True)
3445    p.add_argument('-u', '--user-password', help='password set for this user', required=True)
3446    p.set_defaults(func=bdev_opal_new_user)
3447
3448    def bdev_opal_set_lock_state(args):
3449        rpc.bdev.bdev_opal_set_lock_state(args.client,
3450                                          bdev_name=args.bdev_name,
3451                                          user_id=args.user_id,
3452                                          password=args.password,
3453                                          lock_state=args.lock_state)
3454
3455    p = subparsers.add_parser('bdev_opal_set_lock_state', help="""set lock state for an opal bdev""")
3456    p.add_argument('-b', '--bdev-name', help='opal bdev', required=True)
3457    p.add_argument('-i', '--user-id', help='ID of the user who want to set lock state, either admin or a user assigned to this bdev',
3458                   type=int, required=True)
3459    p.add_argument('-p', '--password', help='password of this user', required=True)
3460    p.add_argument('-l', '--lock-state', help='lock state to set, choose from {readwrite, readonly, rwlock}', required=True)
3461    p.set_defaults(func=bdev_opal_set_lock_state)
3462
3463    # bdev_nvme_send_cmd
3464    def bdev_nvme_send_cmd(args):
3465        print_dict(rpc.nvme.bdev_nvme_send_cmd(args.client,
3466                                               name=args.nvme_name,
3467                                               cmd_type=args.cmd_type,
3468                                               data_direction=args.data_direction,
3469                                               cmdbuf=args.cmdbuf,
3470                                               data=args.data,
3471                                               metadata=args.metadata,
3472                                               data_len=args.data_length,
3473                                               metadata_len=args.metadata_length,
3474                                               timeout_ms=args.timeout_ms))
3475
3476    p = subparsers.add_parser('bdev_nvme_send_cmd', help='NVMe passthrough cmd.')
3477    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""")
3478    p.add_argument('-t', '--cmd-type', help="""Type of nvme cmd. Valid values are: admin, io""")
3479    p.add_argument('-r', '--data-direction', help="""Direction of data transfer. Valid values are: c2h, h2c""")
3480    p.add_argument('-c', '--cmdbuf', help="""NVMe command encoded by base64 urlsafe""")
3481    p.add_argument('-d', '--data', help="""Data transferring to controller from host, encoded by base64 urlsafe""")
3482    p.add_argument('-m', '--metadata', help="""Metadata transferring to controller from host, encoded by base64 urlsafe""")
3483    p.add_argument('-D', '--data-length', help="""Data length required to transfer from controller to host""", type=int)
3484    p.add_argument('-M', '--metadata-length', help="""Metadata length required to transfer from controller to host""", type=int)
3485    p.add_argument('-T', '--timeout-ms',
3486                   help="""Command execution timeout value, in milliseconds,  if 0, don't track timeout""", type=int)
3487    p.set_defaults(func=bdev_nvme_send_cmd)
3488
3489    # Notifications
3490    def notify_get_types(args):
3491        print_dict(rpc.notify.notify_get_types(args.client))
3492
3493    p = subparsers.add_parser('notify_get_types', help='List available notifications that user can subscribe to.')
3494    p.set_defaults(func=notify_get_types)
3495
3496    def notify_get_notifications(args):
3497        ret = rpc.notify.notify_get_notifications(args.client,
3498                                                  id=args.id,
3499                                                  max=args.max)
3500        print_dict(ret)
3501
3502    p = subparsers.add_parser('notify_get_notifications', help='Get notifications')
3503    p.add_argument('-i', '--id', help="""First ID to start fetching from""", type=int)
3504    p.add_argument('-n', '--max', help="""Maximum number of notifications to return in response""", type=int)
3505    p.set_defaults(func=notify_get_notifications)
3506
3507    def thread_get_stats(args):
3508        print_dict(rpc.app.thread_get_stats(args.client))
3509
3510    p = subparsers.add_parser(
3511        'thread_get_stats', help='Display current statistics of all the threads')
3512    p.set_defaults(func=thread_get_stats)
3513
3514    def thread_set_cpumask(args):
3515        ret = rpc.app.thread_set_cpumask(args.client,
3516                                         id=args.id,
3517                                         cpumask=args.cpumask)
3518    p = subparsers.add_parser('thread_set_cpumask',
3519                              help="""set the cpumask of the thread whose ID matches to the
3520    specified value. The thread may be migrated to one of the specified CPUs.""")
3521    p.add_argument('-i', '--id', type=int, help='thread ID')
3522    p.add_argument('-m', '--cpumask', help='cpumask for this thread')
3523    p.set_defaults(func=thread_set_cpumask)
3524
3525    def log_enable_timestamps(args):
3526        ret = rpc.app.log_enable_timestamps(args.client,
3527                                            enabled=args.enabled)
3528    p = subparsers.add_parser('log_enable_timestamps',
3529                              help='Enable or disable timestamps.')
3530    p.add_argument('-d', '--disable', dest='enabled', default=False, action='store_false', help="Disable timestamps")
3531    p.add_argument('-e', '--enable', dest='enabled', action='store_true', help="Enable timestamps")
3532    p.set_defaults(func=log_enable_timestamps)
3533
3534    def thread_get_pollers(args):
3535        print_dict(rpc.app.thread_get_pollers(args.client))
3536
3537    p = subparsers.add_parser(
3538        'thread_get_pollers', help='Display current pollers of all the threads')
3539    p.set_defaults(func=thread_get_pollers)
3540
3541    def thread_get_io_channels(args):
3542        print_dict(rpc.app.thread_get_io_channels(args.client))
3543
3544    p = subparsers.add_parser(
3545        'thread_get_io_channels', help='Display current IO channels of all the threads')
3546    p.set_defaults(func=thread_get_io_channels)
3547
3548    def env_dpdk_get_mem_stats(args):
3549        print_dict(rpc.env_dpdk.env_dpdk_get_mem_stats(args.client))
3550
3551    p = subparsers.add_parser(
3552        'env_dpdk_get_mem_stats', help='write the dpdk memory stats to a file.')
3553    p.set_defaults(func=env_dpdk_get_mem_stats)
3554
3555    # blobfs
3556    def blobfs_detect(args):
3557        print(rpc.blobfs.blobfs_detect(args.client,
3558                                       bdev_name=args.bdev_name))
3559
3560    p = subparsers.add_parser('blobfs_detect', help='Detect whether a blobfs exists on bdev')
3561    p.add_argument('bdev_name', help='Blockdev name to detect blobfs. Example: Malloc0.')
3562    p.set_defaults(func=blobfs_detect)
3563
3564    def blobfs_create(args):
3565        print(rpc.blobfs.blobfs_create(args.client,
3566                                       bdev_name=args.bdev_name,
3567                                       cluster_sz=args.cluster_sz))
3568
3569    p = subparsers.add_parser('blobfs_create', help='Build a blobfs on bdev')
3570    p.add_argument('bdev_name', help='Blockdev name to build blobfs. Example: Malloc0.')
3571    p.add_argument('-c', '--cluster-sz',
3572                   help="""Size of cluster in bytes (Optional). Must be multiple of 4KB page size. Default and minimal value is 1M.""")
3573    p.set_defaults(func=blobfs_create)
3574
3575    def blobfs_mount(args):
3576        print(rpc.blobfs.blobfs_mount(args.client,
3577                                      bdev_name=args.bdev_name,
3578                                      mountpoint=args.mountpoint))
3579
3580    p = subparsers.add_parser('blobfs_mount', help='Mount a blobfs on bdev to host path by FUSE')
3581    p.add_argument('bdev_name', help='Blockdev name where the blobfs is. Example: Malloc0.')
3582    p.add_argument('mountpoint', help='Mountpoint path in host to mount blobfs. Example: /mnt/.')
3583    p.set_defaults(func=blobfs_mount)
3584
3585    def blobfs_set_cache_size(args):
3586        print(rpc.blobfs.blobfs_set_cache_size(args.client,
3587                                               size_in_mb=args.size_in_mb))
3588
3589    p = subparsers.add_parser('blobfs_set_cache_size', help='Set cache size for blobfs')
3590    p.add_argument('size_in_mb', help='Cache size for blobfs in megabytes.', type=int)
3591    p.set_defaults(func=blobfs_set_cache_size)
3592
3593    # fsdev
3594    def fsdev_get_opts(args):
3595        print_json(rpc.fsdev.fsdev_get_opts(args.client))
3596
3597    p = subparsers.add_parser('fsdev_get_opts', help='Get the fsdev subsystem options')
3598    p.set_defaults(func=fsdev_get_opts)
3599
3600    def fsdev_set_opts(args):
3601        print(rpc.fsdev.fsdev_set_opts(args.client, fsdev_io_pool_size=args.fsdev_io_pool_size,
3602                                       fsdev_io_cache_size=args.fsdev_io_cache_size))
3603
3604    p = subparsers.add_parser('fsdev_set_opts', help='Set the fsdev subsystem options')
3605    p.add_argument('fsdev-io-pool-size', help='Size of fsdev IO objects pool', type=int)
3606    p.add_argument('fsdev-io-cache-size', help='Size of fsdev IO objects cache per thread', type=int)
3607    p.set_defaults(func=fsdev_set_opts)
3608
3609    def fsdev_aio_create(args):
3610        print(rpc.fsdev.fsdev_aio_create(args.client, name=args.name, root_path=args.root_path,
3611                                         enable_xattr=args.enable_xattr, enable_writeback_cache=args.enable_writeback_cache,
3612                                         max_write=args.max_write))
3613
3614    p = subparsers.add_parser('fsdev_aio_create', help='Create a aio filesystem')
3615    p.add_argument('name', help='Filesystem name. Example: aio0.')
3616    p.add_argument('root_path', help='Path on the system fs to expose as SPDK filesystem')
3617
3618    group = p.add_mutually_exclusive_group()
3619    group.add_argument('--enable-xattr', help='Enable extended attributes', action='store_true', default=None)
3620    group.add_argument('--disable-xattr', help='Disable extended attributes', dest='enable_xattr', action='store_false', default=None)
3621
3622    group = p.add_mutually_exclusive_group()
3623    group.add_argument('--enable-writeback-cache', help='Enable writeback cache', action='store_true', default=None)
3624    group.add_argument('--disable-writeback-cache', help='Disable writeback cache', dest='enable_writeback_cache', action='store_false',
3625                       default=None)
3626
3627    p.add_argument('-w', '--max-write', help='Max write size in bytes', type=int)
3628    p.set_defaults(func=fsdev_aio_create)
3629
3630    def fsdev_aio_delete(args):
3631        print(rpc.fsdev.fsdev_aio_delete(args.client, name=args.name))
3632
3633    p = subparsers.add_parser('fsdev_aio_delete', help='Delete a aio filesystem')
3634    p.add_argument('name', help='Filesystem name. Example: aio0.')
3635    p.set_defaults(func=fsdev_aio_delete)
3636
3637    # sock
3638    def sock_impl_get_options(args):
3639        print_json(rpc.sock.sock_impl_get_options(args.client,
3640                                                  impl_name=args.impl))
3641
3642    p = subparsers.add_parser('sock_impl_get_options', help="""Get options of socket layer implementation""")
3643    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3644    p.set_defaults(func=sock_impl_get_options)
3645
3646    def sock_impl_set_options(args):
3647        rpc.sock.sock_impl_set_options(args.client,
3648                                       impl_name=args.impl,
3649                                       recv_buf_size=args.recv_buf_size,
3650                                       send_buf_size=args.send_buf_size,
3651                                       enable_recv_pipe=args.enable_recv_pipe,
3652                                       enable_quickack=args.enable_quickack,
3653                                       enable_placement_id=args.enable_placement_id,
3654                                       enable_zerocopy_send_server=args.enable_zerocopy_send_server,
3655                                       enable_zerocopy_send_client=args.enable_zerocopy_send_client,
3656                                       zerocopy_threshold=args.zerocopy_threshold,
3657                                       tls_version=args.tls_version,
3658                                       enable_ktls=args.enable_ktls)
3659
3660    p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""")
3661    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3662    p.add_argument('-r', '--recv-buf-size', help='Size of receive buffer on socket in bytes', type=int)
3663    p.add_argument('-s', '--send-buf-size', help='Size of send buffer on socket in bytes', type=int)
3664    p.add_argument('-p', '--enable-placement-id', help='Option for placement-id. 0:disable,1:incoming_napi,2:incoming_cpu', type=int)
3665    p.add_argument('--enable-recv-pipe', help='Enable receive pipe',
3666                   action='store_true', dest='enable_recv_pipe')
3667    p.add_argument('--disable-recv-pipe', help='Disable receive pipe',
3668                   action='store_false', dest='enable_recv_pipe')
3669    p.add_argument('--enable-quickack', help='Enable quick ACK',
3670                   action='store_true', dest='enable_quickack')
3671    p.add_argument('--disable-quickack', help='Disable quick ACK',
3672                   action='store_false', dest='enable_quickack')
3673    p.add_argument('--enable-zerocopy-send-server', help='Enable zerocopy on send for server sockets',
3674                   action='store_true', dest='enable_zerocopy_send_server')
3675    p.add_argument('--disable-zerocopy-send-server', help='Disable zerocopy on send for server sockets',
3676                   action='store_false', dest='enable_zerocopy_send_server')
3677    p.add_argument('--enable-zerocopy-send-client', help='Enable zerocopy on send for client sockets',
3678                   action='store_true', dest='enable_zerocopy_send_client')
3679    p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets',
3680                   action='store_false', dest='enable_zerocopy_send_client')
3681    p.add_argument('--zerocopy-threshold', help='Set zerocopy_threshold in bytes', type=int)
3682    p.add_argument('--tls-version', help='TLS protocol version', type=int)
3683    p.add_argument('--enable-ktls', help='Enable Kernel TLS',
3684                   action='store_true', dest='enable_ktls')
3685    p.add_argument('--disable-ktls', help='Disable Kernel TLS',
3686                   action='store_false', dest='enable_ktls')
3687    p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_quickack=None,
3688                   enable_placement_id=None, enable_zerocopy_send_server=None, enable_zerocopy_send_client=None,
3689                   zerocopy_threshold=None, tls_version=None, enable_ktls=None)
3690
3691    def sock_set_default_impl(args):
3692        print_json(rpc.sock.sock_set_default_impl(args.client,
3693                                                  impl_name=args.impl))
3694
3695    p = subparsers.add_parser('sock_set_default_impl', help="""Set the default sock implementation""")
3696    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3697    p.set_defaults(func=sock_set_default_impl)
3698
3699    def sock_get_default_impl(args):
3700        print_json(rpc.sock.sock_get_default_impl(args.client))
3701
3702    p = subparsers.add_parser('sock_get_default_impl', help="Get the default sock implementation name")
3703    p.set_defaults(func=sock_get_default_impl)
3704
3705    def framework_get_pci_devices(args):
3706        def splitbuf(buf, step):
3707            return [buf[i:i+step] for i in range(0, len(buf), step)]
3708
3709        devices = rpc.subsystem.framework_get_pci_devices(args.client)
3710        if not args.format_lspci:
3711            print_json(devices)
3712        else:
3713            for devid, dev in enumerate(devices):
3714                print('{} device #{}'.format(dev['address'], devid))
3715                for lineid, line in enumerate(splitbuf(dev['config_space'], 32)):
3716                    print('{:02x}: {}'.format(lineid * 16, ' '.join(splitbuf(line.lower(), 2))))
3717                print()
3718
3719    p = subparsers.add_parser('framework_get_pci_devices', help='''Get a list of attached PCI devices''')
3720    p.add_argument('--format-lspci', help='Format the output in a way to be consumed by lspci -F',
3721                   action='store_true')
3722    p.set_defaults(func=framework_get_pci_devices)
3723
3724    # bdev_nvme_add_error_injection
3725    def bdev_nvme_add_error_injection(args):
3726        print_dict(rpc.nvme.bdev_nvme_add_error_injection(args.client,
3727                                                          name=args.nvme_name,
3728                                                          cmd_type=args.cmd_type,
3729                                                          opc=args.opc,
3730                                                          do_not_submit=args.do_not_submit,
3731                                                          timeout_in_us=args.timeout_in_us,
3732                                                          err_count=args.err_count,
3733                                                          sct=args.sct,
3734                                                          sc=args.sc))
3735    p = subparsers.add_parser('bdev_nvme_add_error_injection',
3736                              help='Add a NVMe command error injection.')
3737    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""", required=True)
3738    p.add_argument('-t', '--cmd-type', help="""Type of NVMe command. Valid values are: admin, io""", required=True)
3739    p.add_argument('-o', '--opc', help="""Opcode of the NVMe command.""", required=True, type=int)
3740    p.add_argument('-s', '--do-not-submit',
3741                   help="""Set to true if request should not be submitted to the controller""",
3742                   dest="do_not_submit", action='store_true')
3743    p.add_argument('-w', '--timeout-in-us', help="""Wait specified microseconds when do_not_submit is true""", type=int)
3744    p.add_argument('-e', '--err-count', help="""Number of matching NVMe commands to inject errors""", type=int)
3745    p.add_argument('-u', '--sct', help="""Status code type""", type=int)
3746    p.add_argument('-c', '--sc', help="""Status code""", type=int)
3747    p.set_defaults(func=bdev_nvme_add_error_injection)
3748
3749    # bdev_nvme_remove_error_injection
3750    def bdev_nvme_remove_error_injection(args):
3751        print_dict(rpc.nvme.bdev_nvme_remove_error_injection(args.client,
3752                                                             name=args.nvme_name,
3753                                                             cmd_type=args.cmd_type,
3754                                                             opc=args.opc))
3755    p = subparsers.add_parser('bdev_nvme_remove_error_injection',
3756                              help='Removes a NVMe command error injection.')
3757    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""", required=True)
3758    p.add_argument('-t', '--cmd-type', help="""Type of nvme cmd. Valid values are: admin, io""", required=True)
3759    p.add_argument('-o', '--opc', help="""Opcode of the nvme cmd.""", required=True, type=int)
3760    p.set_defaults(func=bdev_nvme_remove_error_injection)
3761
3762    def bdev_daos_create(args):
3763        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
3764        print_json(rpc.bdev.bdev_daos_create(args.client,
3765                                             num_blocks=int(num_blocks),
3766                                             block_size=args.block_size,
3767                                             name=args.name,
3768                                             uuid=args.uuid,
3769                                             pool=args.pool,
3770                                             cont=args.cont,
3771                                             oclass=args.oclass))
3772    p = subparsers.add_parser('bdev_daos_create',
3773                              help='Create a bdev with DAOS backend')
3774    p.add_argument('name', help="Name of the bdev")
3775    p.add_argument('pool', help="UUID of the DAOS pool")
3776    p.add_argument('cont', help="UUID of the DAOS container")
3777    p.add_argument(
3778        'total_size', help='Size of DAOS bdev in MB (float > 0)', type=float)
3779    p.add_argument('block_size', help='Block size for this bdev', type=int)
3780    p.add_argument('-u', '--uuid', help="UUID of the bdev")
3781    p.add_argument('-o', '--oclass', help="DAOS object class")
3782    p.set_defaults(func=bdev_daos_create)
3783
3784    def bdev_daos_delete(args):
3785        rpc.bdev.bdev_daos_delete(args.client,
3786                                  name=args.name)
3787
3788    p = subparsers.add_parser('bdev_daos_delete',
3789                              help='Delete a DAOS disk')
3790    p.add_argument('name', help='DAOS bdev name')
3791    p.set_defaults(func=bdev_daos_delete)
3792
3793    def bdev_daos_resize(args):
3794        print_json(rpc.bdev.bdev_daos_resize(args.client,
3795                                             name=args.name,
3796                                             new_size=int(args.new_size)))
3797
3798    p = subparsers.add_parser('bdev_daos_resize',
3799                              help='Resize a DAOS bdev')
3800    p.add_argument('name', help='DAOS bdev name')
3801    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
3802    p.set_defaults(func=bdev_daos_resize)
3803
3804    def iobuf_set_options(args):
3805        rpc.iobuf.iobuf_set_options(args.client,
3806                                    small_pool_count=args.small_pool_count,
3807                                    large_pool_count=args.large_pool_count,
3808                                    small_bufsize=args.small_bufsize,
3809                                    large_bufsize=args.large_bufsize)
3810    p = subparsers.add_parser('iobuf_set_options', help='Set iobuf pool options')
3811    p.add_argument('--small-pool-count', help='number of small buffers in the global pool', type=int)
3812    p.add_argument('--large-pool-count', help='number of large buffers in the global pool', type=int)
3813    p.add_argument('--small-bufsize', help='size of a small buffer', type=int)
3814    p.add_argument('--large-bufsize', help='size of a large buffer', type=int)
3815    p.set_defaults(func=iobuf_set_options)
3816
3817    def iobuf_get_stats(args):
3818        print_dict(rpc.iobuf.iobuf_get_stats(args.client))
3819
3820    p = subparsers.add_parser('iobuf_get_stats', help='Display iobuf statistics')
3821    p.set_defaults(func=iobuf_get_stats)
3822
3823    def bdev_nvme_start_mdns_discovery(args):
3824        rpc.bdev.bdev_nvme_start_mdns_discovery(args.client,
3825                                                name=args.name,
3826                                                svcname=args.svcname,
3827                                                hostnqn=args.hostnqn)
3828
3829    p = subparsers.add_parser('bdev_nvme_start_mdns_discovery', help='Start mdns based automatic discovery')
3830    p.add_argument('-b', '--name', help="Name of the NVMe controller prefix for each bdev name", required=True)
3831    p.add_argument('-s', '--svcname', help='Service type to discover: e.g., _nvme-disc._tcp', required=True)
3832    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
3833    p.set_defaults(func=bdev_nvme_start_mdns_discovery)
3834
3835    def bdev_nvme_stop_mdns_discovery(args):
3836        rpc.bdev.bdev_nvme_stop_mdns_discovery(args.client, name=args.name)
3837
3838    p = subparsers.add_parser('bdev_nvme_stop_mdns_discovery', help='Stop automatic mdns discovery')
3839    p.add_argument('-b', '--name', help="Name of the service to stop", required=True)
3840    p.set_defaults(func=bdev_nvme_stop_mdns_discovery)
3841
3842    def bdev_nvme_get_mdns_discovery_info(args):
3843        print_dict(rpc.bdev.bdev_nvme_get_mdns_discovery_info(args.client))
3844
3845    p = subparsers.add_parser('bdev_nvme_get_mdns_discovery_info', help='Get information about the automatic mdns discovery')
3846    p.set_defaults(func=bdev_nvme_get_mdns_discovery_info)
3847
3848    def keyring_file_add_key(args):
3849        rpc.keyring.keyring_file_add_key(args.client, args.name, args.path)
3850
3851    p = subparsers.add_parser('keyring_file_add_key', help='Add a file-based key to the keyring')
3852    p.add_argument('name', help='Name of the key to add')
3853    p.add_argument('path', help='Path of the file containing the key')
3854    p.set_defaults(func=keyring_file_add_key)
3855
3856    def keyring_file_remove_key(args):
3857        rpc.keyring.keyring_file_remove_key(args.client, args.name)
3858
3859    p = subparsers.add_parser('keyring_file_remove_key', help='Remove a file-based key from the keyring')
3860    p.add_argument('name', help='Name of the key to remove')
3861    p.set_defaults(func=keyring_file_remove_key)
3862
3863    def keyring_get_keys(args):
3864        print_dict(rpc.keyring.keyring_get_keys(args.client))
3865
3866    p = subparsers.add_parser('keyring_get_keys', help='Get a list of registered keys')
3867    p.set_defaults(func=keyring_get_keys)
3868
3869    def keyring_linux_set_options(args):
3870        rpc.keyring.keyring_linux_set_options(args.client, args.enable)
3871
3872    p = subparsers.add_parser('keyring_linux_set_options', help='Set options of the keyring_linux module')
3873    p.add_argument('-e', '--enable', help='Enable keyring_linux module', action='store_true')
3874    p.set_defaults(func=keyring_linux_set_options)
3875
3876    def check_called_name(name):
3877        if name in deprecated_aliases:
3878            print("{} is deprecated, use {} instead.".format(name, deprecated_aliases[name]), file=sys.stderr)
3879
3880    class dry_run_client:
3881        def call(self, method, params=None):
3882            print("Request:\n" + json.dumps({"method": method, "params": params}, indent=2))
3883
3884    def null_print(arg):
3885        pass
3886
3887    def call_rpc_func(args):
3888        args.func(args)
3889        check_called_name(args.called_rpc_name)
3890
3891    def execute_script(parser, client, timeout, fd):
3892        executed_rpc = ""
3893        for rpc_call in map(str.rstrip, fd):
3894            if not rpc_call.strip():
3895                continue
3896            executed_rpc = "\n".join([executed_rpc, rpc_call])
3897            rpc_args = shlex.split(rpc_call)
3898            if rpc_args[0][0] == '#':
3899                # Ignore lines starting with # - treat them as comments
3900                continue
3901            args = parser.parse_args(rpc_args)
3902            args.client = client
3903            args.timeout = timeout
3904            try:
3905                call_rpc_func(args)
3906            except JSONRPCException as ex:
3907                print("Exception:")
3908                print(executed_rpc.strip() + " <<<")
3909                print(ex.message)
3910                exit(1)
3911
3912    def load_plugin(args):
3913        # Create temporary parser, pull out the plugin parameter, load the module, and then run the real argument parser
3914        plugin_parser = argparse.ArgumentParser(add_help=False)
3915        plugin_parser.add_argument('--plugin', dest='rpc_plugin', help='Module name of plugin with additional RPC commands')
3916
3917        rpc_module = plugin_parser.parse_known_args()[0].rpc_plugin
3918        if args is not None:
3919            rpc_module = plugin_parser.parse_known_args(args)[0].rpc_plugin
3920
3921        if rpc_module in plugins:
3922            return
3923
3924        if rpc_module is not None:
3925            try:
3926                rpc_plugin = importlib.import_module(rpc_module)
3927                try:
3928                    rpc_plugin.spdk_rpc_plugin_initialize(subparsers)
3929                    plugins.append(rpc_module)
3930                except AttributeError:
3931                    print("Module %s does not contain 'spdk_rpc_plugin_initialize' function" % rpc_module)
3932            except ModuleNotFoundError:
3933                print("Module %s not found" % rpc_module)
3934
3935    def replace_arg_underscores(args):
3936        # All option names are defined with dashes only - for example: --tgt-name
3937        # But if user used underscores, convert them to dashes (--tgt_name => --tgt-name)
3938        # SPDK was inconsistent previously and had some options with underscores, so
3939        # doing this conversion ensures backward compatibility with older scripts.
3940        for i in range(len(args)):
3941            arg = args[i]
3942            if arg.startswith('--') and "_" in arg:
3943                opt, *vals = arg.split('=')
3944                args[i] = '='.join([opt.replace('_', '-'), *vals])
3945
3946    plugins = []
3947    load_plugin(None)
3948
3949    replace_arg_underscores(sys.argv)
3950
3951    args = parser.parse_args()
3952
3953    try:
3954        use_go_client = int(os.getenv('SPDK_JSONRPC_GO_CLIENT', 0)) == 1
3955    except ValueError:
3956        use_go_client = False
3957
3958    if sys.stdin.isatty() and not hasattr(args, 'func'):
3959        # No arguments and no data piped through stdin
3960        parser.print_help()
3961        exit(1)
3962    if args.is_server:
3963        for input in sys.stdin:
3964            cmd = shlex.split(input)
3965            replace_arg_underscores(cmd)
3966            try:
3967                load_plugin(cmd)
3968                tmp_args = parser.parse_args(cmd)
3969            except SystemExit as ex:
3970                print("**STATUS=1", flush=True)
3971                continue
3972
3973            try:
3974                if use_go_client:
3975                    tmp_args.client = rpc.client.JSONRPCGoClient(tmp_args.server_addr,
3976                                                                 log_level=getattr(logging, tmp_args.verbose.upper()))
3977                else:
3978                    tmp_args.client = rpc.client.JSONRPCClient(
3979                        tmp_args.server_addr, tmp_args.port, tmp_args.timeout,
3980                        log_level=getattr(logging, tmp_args.verbose.upper()), conn_retries=tmp_args.conn_retries)
3981                call_rpc_func(tmp_args)
3982                print("**STATUS=0", flush=True)
3983            except JSONRPCException as ex:
3984                print(ex.message)
3985                print("**STATUS=1", flush=True)
3986        exit(0)
3987    elif args.dry_run:
3988        args.client = dry_run_client()
3989        print_dict = null_print
3990        print_json = null_print
3991        print_array = null_print
3992    elif args.go_client or use_go_client:
3993        try:
3994            args.client = rpc.client.JSONRPCGoClient(args.server_addr,
3995                                                     log_level=getattr(logging, args.verbose.upper()))
3996        except JSONRPCException as ex:
3997            print(ex.message)
3998            exit(1)
3999    else:
4000        try:
4001            args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.timeout,
4002                                                   log_level=getattr(logging, args.verbose.upper()),
4003                                                   conn_retries=args.conn_retries)
4004        except JSONRPCException as ex:
4005            print(ex.message)
4006            exit(1)
4007
4008    if hasattr(args, 'func'):
4009        try:
4010            call_rpc_func(args)
4011        except JSONRPCException as ex:
4012            print(ex.message)
4013            exit(1)
4014    else:
4015        execute_script(parser, args.client, args.timeout, sys.stdin)
4016