xref: /spdk/scripts/rpc.py (revision 60982c759db49b4f4579f16e3b24df0725ba4b94)
1#!/usr/bin/env python3
2#  SPDX-License-Identifier: BSD-3-Clause
3#  Copyright (C) 2016 Intel Corporation
4#  All rights reserved.
5#  Copyright (c) 2022 Dell Inc, or its subsidiaries.
6#  Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
7#
8
9import logging
10import argparse
11import importlib
12import os
13import sys
14import shlex
15import json
16
17try:
18    from shlex import quote
19except ImportError:
20    from pipes import quote
21
22sys.path.append(os.path.dirname(__file__) + '/../python')
23
24import spdk.rpc as rpc  # noqa
25from spdk.rpc.client import print_dict, print_json, JSONRPCException  # noqa
26from spdk.rpc.helpers import deprecated_aliases  # noqa
27
28
29def print_array(a):
30    print(" ".join((quote(v) for v in a)))
31
32
33if __name__ == "__main__":
34    parser = argparse.ArgumentParser(
35        description='SPDK RPC command line interface', usage='%(prog)s [options]')
36    parser.add_argument('-s', dest='server_addr',
37                        help='RPC domain socket path or IP address', default='/var/tmp/spdk.sock')
38    parser.add_argument('-p', dest='port',
39                        help='RPC port number (if server_addr is IP address)',
40                        default=5260, type=int)
41    parser.add_argument('-t', dest='timeout',
42                        help='Timeout as a floating point number expressed in seconds waiting for response. Default: 60.0',
43                        default=60.0, type=float)
44    parser.add_argument('-r', dest='conn_retries',
45                        help='Retry connecting to the RPC server N times with 0.2s interval. Default: 0',
46                        default=0, type=int)
47    parser.add_argument('-v', dest='verbose', action='store_const', const="INFO",
48                        help='Set verbose mode to INFO', default="ERROR")
49    parser.add_argument('--verbose', dest='verbose', choices=['DEBUG', 'INFO', 'ERROR'],
50                        help="""Set verbose level. """)
51    parser.add_argument('--dry-run', dest='dry_run', action='store_true', help="Display request and exit")
52    parser.set_defaults(dry_run=False)
53    parser.add_argument('--server', dest='is_server', action='store_true',
54                        help="Start listening on stdin, parse each line as a regular rpc.py execution and create \
55                                a separate connection for each command. Each command's output ends with either \
56                                **STATUS=0 if the command succeeded or **STATUS=1 if it failed. --server is meant \
57                                to be used in conjunction with bash coproc, where stdin and stdout are connected to \
58                                pipes and can be used as a faster way to send RPC commands. If enabled, rpc.py \
59                                must be executed without any other parameters.")
60    parser.set_defaults(is_server=False)
61    parser.add_argument('--plugin', dest='rpc_plugin', help='Module name of plugin with additional RPC commands')
62    subparsers = parser.add_subparsers(help='RPC methods', dest='called_rpc_name', metavar='')
63
64    def framework_start_init(args):
65        rpc.framework_start_init(args.client)
66
67    p = subparsers.add_parser('framework_start_init', help='Start initialization of subsystems')
68    p.set_defaults(func=framework_start_init)
69
70    def framework_wait_init(args):
71        rpc.framework_wait_init(args.client)
72
73    p = subparsers.add_parser('framework_wait_init', help='Block until subsystems have been initialized')
74    p.set_defaults(func=framework_wait_init)
75
76    def rpc_get_methods(args):
77        print_dict(rpc.rpc_get_methods(args.client,
78                                       current=args.current,
79                                       include_aliases=args.include_aliases))
80
81    p = subparsers.add_parser('rpc_get_methods', help='Get list of supported RPC methods')
82    p.add_argument('-c', '--current', help='Get list of RPC methods only callable in the current state.', action='store_true')
83    p.add_argument('-i', '--include-aliases', help='include RPC aliases', action='store_true')
84    p.set_defaults(func=rpc_get_methods)
85
86    def spdk_get_version(args):
87        print_json(rpc.spdk_get_version(args.client))
88
89    p = subparsers.add_parser('spdk_get_version', help='Get SPDK version')
90    p.set_defaults(func=spdk_get_version)
91
92    def save_config(args):
93        rpc.save_config(args.client,
94                        sys.stdout,
95                        indent=args.indent)
96
97    p = subparsers.add_parser('save_config', help="""Write current (live) configuration of SPDK subsystems and targets to stdout.
98    """)
99    p.add_argument('-i', '--indent', help="""Indent level. Value less than 0 mean compact mode. Default indent level is 2.
100    """, type=int, default=2)
101    p.set_defaults(func=save_config)
102
103    def load_config(args):
104        rpc.load_config(args.client, args.json_conf,
105                        include_aliases=args.include_aliases)
106
107    p = subparsers.add_parser('load_config', help="""Configure SPDK subsystems and targets using JSON RPC.""")
108    p.add_argument('-i', '--include-aliases', help='include RPC aliases', action='store_true')
109    p.add_argument('-j', '--json-conf', help='Valid JSON configuration', default=sys.stdin)
110    p.set_defaults(func=load_config)
111
112    def save_subsystem_config(args):
113        rpc.save_subsystem_config(args.client,
114                                  sys.stdout,
115                                  indent=args.indent,
116                                  name=args.name)
117
118    p = subparsers.add_parser('save_subsystem_config', help="""Write current (live) configuration of SPDK subsystem to stdout.
119    """)
120    p.add_argument('-i', '--indent', help="""Indent level. Value less than 0 mean compact mode. Default indent level is 2.
121    """, type=int, default=2)
122    p.add_argument('-n', '--name', help='Name of subsystem', required=True)
123    p.set_defaults(func=save_subsystem_config)
124
125    def load_subsystem_config(args):
126        rpc.load_subsystem_config(args.client,
127                                  args.json_conf)
128
129    p = subparsers.add_parser('load_subsystem_config', help="""Configure SPDK subsystem using JSON RPC.""")
130    p.add_argument('-j', '--json-conf', help='Valid JSON configuration', default=sys.stdin)
131    p.set_defaults(func=load_subsystem_config)
132
133    # app
134    def spdk_kill_instance(args):
135        rpc.app.spdk_kill_instance(args.client,
136                                   sig_name=args.sig_name)
137
138    p = subparsers.add_parser('spdk_kill_instance', help='Send signal to instance')
139    p.add_argument('sig_name', help='signal will be sent to server.')
140    p.set_defaults(func=spdk_kill_instance)
141
142    def framework_monitor_context_switch(args):
143        enabled = None
144        if args.enable:
145            enabled = True
146        if args.disable:
147            enabled = False
148        print_dict(rpc.app.framework_monitor_context_switch(args.client,
149                                                            enabled=enabled))
150
151    p = subparsers.add_parser('framework_monitor_context_switch',
152                              help='Control whether the context switch monitor is enabled')
153    p.add_argument('-e', '--enable', action='store_true', help='Enable context switch monitoring')
154    p.add_argument('-d', '--disable', action='store_true', help='Disable context switch monitoring')
155    p.set_defaults(func=framework_monitor_context_switch)
156
157    def framework_get_reactors(args):
158        print_dict(rpc.app.framework_get_reactors(args.client))
159
160    p = subparsers.add_parser(
161        'framework_get_reactors', help='Display list of all reactors')
162    p.set_defaults(func=framework_get_reactors)
163
164    def framework_set_scheduler(args):
165        rpc.app.framework_set_scheduler(args.client,
166                                        name=args.name,
167                                        period=args.period,
168                                        load_limit=args.load_limit,
169                                        core_limit=args.core_limit,
170                                        core_busy=args.core_busy)
171
172    p = subparsers.add_parser(
173        'framework_set_scheduler', help='Select thread scheduler that will be activated and its period (experimental)')
174    p.add_argument('name', help="Name of a scheduler")
175    p.add_argument('-p', '--period', help="Scheduler period in microseconds", type=int)
176    p.add_argument('--load-limit', help="Scheduler load limit. Reserved for dynamic scheduler", type=int, required=False)
177    p.add_argument('--core-limit', help="Scheduler core limit. Reserved for dynamic scheduler", type=int, required=False)
178    p.add_argument('--core-busy', help="Scheduler core busy limit. Reserved for dynamic schedler", type=int, required=False)
179    p.set_defaults(func=framework_set_scheduler)
180
181    def framework_get_scheduler(args):
182        print_dict(rpc.app.framework_get_scheduler(args.client))
183
184    p = subparsers.add_parser(
185        'framework_get_scheduler', help='Display currently set scheduler and its properties.')
186    p.set_defaults(func=framework_get_scheduler)
187
188    def framework_disable_cpumask_locks(args):
189        rpc.framework_disable_cpumask_locks(args.client)
190
191    p = subparsers.add_parser('framework_disable_cpumask_locks',
192                              help='Disable CPU core lock files.')
193    p.set_defaults(func=framework_disable_cpumask_locks)
194
195    def framework_enable_cpumask_locks(args):
196        rpc.framework_enable_cpumask_locks(args.client)
197
198    p = subparsers.add_parser('framework_enable_cpumask_locks',
199                              help='Enable CPU core lock files.')
200    p.set_defaults(func=framework_enable_cpumask_locks)
201
202    # bdev
203    def bdev_set_options(args):
204        rpc.bdev.bdev_set_options(args.client,
205                                  bdev_io_pool_size=args.bdev_io_pool_size,
206                                  bdev_io_cache_size=args.bdev_io_cache_size,
207                                  bdev_auto_examine=args.bdev_auto_examine)
208
209    p = subparsers.add_parser('bdev_set_options',
210                              help="""Set options of bdev subsystem""")
211    p.add_argument('-p', '--bdev-io-pool-size', help='Number of bdev_io structures in shared buffer pool', type=int)
212    p.add_argument('-c', '--bdev-io-cache-size', help='Maximum number of bdev_io structures cached per thread', type=int)
213    group = p.add_mutually_exclusive_group()
214    group.add_argument('-e', '--enable-auto-examine', dest='bdev_auto_examine', help='Allow to auto examine', action='store_true')
215    group.add_argument('-d', '--disable-auto-examine', dest='bdev_auto_examine', help='Not allow to auto examine', action='store_false')
216    p.set_defaults(bdev_auto_examine=True)
217    p.set_defaults(func=bdev_set_options)
218
219    def bdev_examine(args):
220        rpc.bdev.bdev_examine(args.client,
221                              name=args.name)
222
223    p = subparsers.add_parser('bdev_examine',
224                              help="""examine a bdev if it exists, or will examine it after it is created""")
225    p.add_argument('-b', '--name', help='Name or alias of the bdev')
226    p.set_defaults(func=bdev_examine)
227
228    def bdev_wait_for_examine(args):
229        rpc.bdev.bdev_wait_for_examine(args.client)
230
231    p = subparsers.add_parser('bdev_wait_for_examine',
232                              help="""Report when all bdevs have been examined""")
233    p.set_defaults(func=bdev_wait_for_examine)
234
235    def bdev_compress_create(args):
236        print_json(rpc.bdev.bdev_compress_create(args.client,
237                                                 base_bdev_name=args.base_bdev_name,
238                                                 pm_path=args.pm_path,
239                                                 lb_size=args.lb_size))
240
241    p = subparsers.add_parser('bdev_compress_create', help='Add a compress vbdev')
242    p.add_argument('-b', '--base-bdev-name', help="Name of the base bdev")
243    p.add_argument('-p', '--pm-path', help="Path to persistent memory")
244    p.add_argument('-l', '--lb-size', help="Compressed vol logical block size (optional, if used must be 512 or 4096)", type=int)
245    p.set_defaults(func=bdev_compress_create)
246
247    def bdev_compress_delete(args):
248        rpc.bdev.bdev_compress_delete(args.client,
249                                      name=args.name)
250
251    p = subparsers.add_parser('bdev_compress_delete', help='Delete a compress disk')
252    p.add_argument('name', help='compress bdev name')
253    p.set_defaults(func=bdev_compress_delete)
254
255    def bdev_compress_get_orphans(args):
256        print_dict(rpc.bdev.bdev_compress_get_orphans(args.client,
257                                                      name=args.name))
258    p = subparsers.add_parser(
259        'bdev_compress_get_orphans', help='Display list of orphaned compress bdevs.')
260    p.add_argument('-b', '--name', help="Name of a comp bdev. Example: COMP_Nvme0n1", required=False)
261    p.set_defaults(func=bdev_compress_get_orphans)
262
263    def bdev_crypto_create(args):
264        print_json(rpc.bdev.bdev_crypto_create(args.client,
265                                               base_bdev_name=args.base_bdev_name,
266                                               name=args.name,
267                                               crypto_pmd=args.crypto_pmd,
268                                               key=args.key,
269                                               cipher=args.cipher,
270                                               key2=args.key2,
271                                               key_name=args.key_name))
272    p = subparsers.add_parser('bdev_crypto_create', help='Add a crypto vbdev')
273    p.add_argument('base_bdev_name', help="Name of the base bdev")
274    p.add_argument('name', help="Name of the crypto vbdev")
275    p.add_argument('-p', '--crypto-pmd', help="Name of the crypto device driver. Obsolete, see dpdk_cryptodev_set_driver", required=False)
276    p.add_argument('-k', '--key', help="Key. Obsolete, see accel_crypto_key_create", required=False)
277    p.add_argument('-c', '--cipher', help="cipher to use. Obsolete, see accel_crypto_key_create", required=False)
278    p.add_argument('-k2', '--key2', help="2nd key for cipher AES_XTS. Obsolete, see accel_crypto_key_create", default=None)
279    p.add_argument('-n', '--key-name', help="Key name to use, see accel_crypto_key_create", required=False)
280    p.set_defaults(func=bdev_crypto_create)
281
282    def bdev_crypto_delete(args):
283        rpc.bdev.bdev_crypto_delete(args.client,
284                                    name=args.name)
285
286    p = subparsers.add_parser('bdev_crypto_delete', help='Delete a crypto disk')
287    p.add_argument('name', help='crypto bdev name')
288    p.set_defaults(func=bdev_crypto_delete)
289
290    def bdev_ocf_create(args):
291        print_json(rpc.bdev.bdev_ocf_create(args.client,
292                                            name=args.name,
293                                            mode=args.mode,
294                                            cache_line_size=args.cache_line_size,
295                                            cache_bdev_name=args.cache_bdev_name,
296                                            core_bdev_name=args.core_bdev_name))
297    p = subparsers.add_parser('bdev_ocf_create', help='Add an OCF block device')
298    p.add_argument('name', help='Name of resulting OCF bdev')
299    p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo'])
300    p.add_argument(
301        '--cache-line-size',
302        help='OCF cache line size. The unit is KiB',
303        type=int,
304        choices=[4, 8, 16, 32, 64],
305        required=False
306    )
307    p.add_argument('cache_bdev_name', help='Name of underlying cache bdev')
308    p.add_argument('core_bdev_name', help='Name of underlying core bdev')
309    p.set_defaults(func=bdev_ocf_create)
310
311    def bdev_ocf_delete(args):
312        rpc.bdev.bdev_ocf_delete(args.client,
313                                 name=args.name)
314
315    p = subparsers.add_parser('bdev_ocf_delete', help='Delete an OCF block device')
316    p.add_argument('name', help='Name of OCF bdev')
317    p.set_defaults(func=bdev_ocf_delete)
318
319    def bdev_ocf_get_stats(args):
320        print_dict(rpc.bdev.bdev_ocf_get_stats(args.client,
321                                               name=args.name))
322    p = subparsers.add_parser('bdev_ocf_get_stats', help='Get statistics of chosen OCF block device')
323    p.add_argument('name', help='Name of OCF bdev')
324    p.set_defaults(func=bdev_ocf_get_stats)
325
326    def bdev_ocf_reset_stats(args):
327        print_dict(rpc.bdev.bdev_ocf_reset_stats(args.client,
328                                                 name=args.name))
329    p = subparsers.add_parser('bdev_ocf_reset_stats', help='Reset statistics of chosen OCF block device')
330    p.add_argument('name', help='Name of OCF bdev')
331    p.set_defaults(func=bdev_ocf_reset_stats)
332
333    def bdev_ocf_get_bdevs(args):
334        print_dict(rpc.bdev.bdev_ocf_get_bdevs(args.client,
335                                               name=args.name))
336    p = subparsers.add_parser('bdev_ocf_get_bdevs', help='Get list of OCF devices including unregistered ones')
337    p.add_argument('name', nargs='?', help='name of OCF vbdev or name of cache device or name of core device (optional)')
338    p.set_defaults(func=bdev_ocf_get_bdevs)
339
340    def bdev_ocf_set_cache_mode(args):
341        print_json(rpc.bdev.bdev_ocf_set_cache_mode(args.client,
342                                                    name=args.name,
343                                                    mode=args.mode))
344    p = subparsers.add_parser('bdev_ocf_set_cache_mode',
345                              help='Set cache mode of OCF block device')
346    p.add_argument('name', help='Name of OCF bdev')
347    p.add_argument('mode', help='OCF cache mode', choices=['wb', 'wt', 'pt', 'wa', 'wi', 'wo'])
348    p.set_defaults(func=bdev_ocf_set_cache_mode)
349
350    def bdev_ocf_set_seqcutoff(args):
351        rpc.bdev.bdev_ocf_set_seqcutoff(args.client,
352                                        name=args.name,
353                                        policy=args.policy,
354                                        threshold=args.threshold,
355                                        promotion_count=args.promotion_count)
356    p = subparsers.add_parser('bdev_ocf_set_seqcutoff',
357                              help='Set sequential cutoff parameters on all cores for the given OCF cache device')
358    p.add_argument('name', help='Name of OCF cache bdev')
359    p.add_argument('-t', '--threshold', type=int,
360                   help='Activation threshold [KiB]')
361    p.add_argument('-c', '--promotion-count', type=int,
362                   help='Promotion request count')
363    p.add_argument('-p', '--policy', choices=['always', 'full', 'never'], required=True,
364                   help='Sequential cutoff policy')
365    p.set_defaults(func=bdev_ocf_set_seqcutoff)
366
367    def bdev_ocf_flush_start(args):
368        rpc.bdev.bdev_ocf_flush_start(args.client, name=args.name)
369    p = subparsers.add_parser('bdev_ocf_flush_start',
370                              help='Start flushing OCF cache device')
371    p.add_argument('name', help='Name of OCF bdev')
372    p.set_defaults(func=bdev_ocf_flush_start)
373
374    def bdev_ocf_flush_status(args):
375        print_json(rpc.bdev.bdev_ocf_flush_status(args.client, name=args.name))
376    p = subparsers.add_parser('bdev_ocf_flush_status',
377                              help='Get flush status of OCF cache device')
378    p.add_argument('name', help='Name of OCF bdev')
379    p.set_defaults(func=bdev_ocf_flush_status)
380
381    def bdev_malloc_create(args):
382        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
383        print_json(rpc.bdev.bdev_malloc_create(args.client,
384                                               num_blocks=int(num_blocks),
385                                               block_size=args.block_size,
386                                               physical_block_size=args.physical_block_size,
387                                               name=args.name,
388                                               uuid=args.uuid,
389                                               optimal_io_boundary=args.optimal_io_boundary,
390                                               md_size=args.md_size,
391                                               md_interleave=args.md_interleave,
392                                               dif_type=args.dif_type,
393                                               dif_is_head_of_md=args.dif_is_head_of_md))
394    p = subparsers.add_parser('bdev_malloc_create', help='Create a bdev with malloc backend')
395    p.add_argument('-b', '--name', help="Name of the bdev")
396    p.add_argument('-u', '--uuid', help="UUID of the bdev")
397    p.add_argument(
398        'total_size', help='Size of malloc bdev in MB (float > 0)', type=float)
399    p.add_argument('block_size', help='Data block size for this bdev', type=int)
400    p.add_argument('-p', '--physical-block-size', help='Physical block size for this bdev.', type=int)
401    p.add_argument('-o', '--optimal-io-boundary', help="""Split on optimal IO boundary, in number of
402    blocks, default 0 (disabled)""", type=int)
403    p.add_argument('-m', '--md-size', type=int,
404                   help='Metadata size for this bdev (0, 8, 16, 32, 64, or 128). Default is 0.')
405    p.add_argument('-i', '--md-interleave', action='store_true',
406                   help='Metadata location, interleaved if set, and separated if omitted.')
407    p.add_argument('-t', '--dif-type', type=int, choices=[0, 1, 2, 3],
408                   help='Protection information type. Parameter --md-size needs'
409                        'to be set along --dif-type. Default=0 - no protection.')
410    p.add_argument('-d', '--dif-is-head-of-md', action='store_true',
411                   help='Protection information is in the first 8 bytes of metadata. Default=false.')
412    p.set_defaults(func=bdev_malloc_create)
413
414    def bdev_malloc_delete(args):
415        rpc.bdev.bdev_malloc_delete(args.client,
416                                    name=args.name)
417
418    p = subparsers.add_parser('bdev_malloc_delete', help='Delete a malloc disk')
419    p.add_argument('name', help='malloc bdev name')
420    p.set_defaults(func=bdev_malloc_delete)
421
422    def bdev_null_create(args):
423        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
424        if args.dif_type and not args.md_size:
425            print("ERROR: --md-size must be > 0 when --dif-type is > 0")
426            exit(1)
427        print_json(rpc.bdev.bdev_null_create(args.client,
428                                             num_blocks=num_blocks,
429                                             block_size=args.block_size,
430                                             physical_block_size=args.physical_block_size,
431                                             name=args.name,
432                                             uuid=args.uuid,
433                                             md_size=args.md_size,
434                                             dif_type=args.dif_type,
435                                             dif_is_head_of_md=args.dif_is_head_of_md))
436
437    p = subparsers.add_parser('bdev_null_create', help='Add a bdev with null backend')
438    p.add_argument('name', help='Block device name')
439    p.add_argument('-u', '--uuid', help='UUID of the bdev')
440    p.add_argument('total_size', help='Size of null bdev in MB (int > 0). Includes only data blocks.', type=int)
441    p.add_argument('block_size', help='Block size for this bdev.'
442                                      'Should be a sum of block size and metadata size if --md-size is used.', type=int)
443    p.add_argument('-p', '--physical-block-size', help='Physical block size for this bdev.', type=int)
444    p.add_argument('-m', '--md-size', type=int,
445                   help='Metadata size for this bdev. Default=0.')
446    p.add_argument('-t', '--dif-type', type=int, choices=[0, 1, 2, 3],
447                   help='Protection information type. Parameter --md-size needs'
448                        'to be set along --dif-type. Default=0 - no protection.')
449    p.add_argument('-d', '--dif-is-head-of-md', action='store_true',
450                   help='Protection information is in the first 8 bytes of metadata. Default=false.')
451    p.set_defaults(func=bdev_null_create)
452
453    def bdev_null_delete(args):
454        rpc.bdev.bdev_null_delete(args.client,
455                                  name=args.name)
456
457    p = subparsers.add_parser('bdev_null_delete', help='Delete a null bdev')
458    p.add_argument('name', help='null bdev name')
459    p.set_defaults(func=bdev_null_delete)
460
461    def bdev_null_resize(args):
462        print_json(rpc.bdev.bdev_null_resize(args.client,
463                                             name=args.name,
464                                             new_size=int(args.new_size)))
465
466    p = subparsers.add_parser('bdev_null_resize',
467                              help='Resize a null bdev')
468    p.add_argument('name', help='null bdev name')
469    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
470    p.set_defaults(func=bdev_null_resize)
471
472    def bdev_aio_create(args):
473        print_json(rpc.bdev.bdev_aio_create(args.client,
474                                            filename=args.filename,
475                                            name=args.name,
476                                            block_size=args.block_size,
477                                            readonly=args.readonly))
478
479    p = subparsers.add_parser('bdev_aio_create', help='Add a bdev with aio backend')
480    p.add_argument('filename', help='Path to device or file (ex: /dev/sda)')
481    p.add_argument('name', help='Block device name')
482    p.add_argument('block_size', help='Block size for this bdev', type=int, nargs='?')
483    p.add_argument("-r", "--readonly", action='store_true', help='Set this bdev as read-only')
484    p.set_defaults(func=bdev_aio_create)
485
486    def bdev_aio_rescan(args):
487        print_json(rpc.bdev.bdev_aio_rescan(args.client,
488                                            name=args.name))
489
490    p = subparsers.add_parser('bdev_aio_rescan', help='Rescan a bdev size with aio backend')
491    p.add_argument('name', help='Block device name')
492    p.set_defaults(func=bdev_aio_rescan)
493
494    def bdev_aio_delete(args):
495        rpc.bdev.bdev_aio_delete(args.client,
496                                 name=args.name)
497
498    p = subparsers.add_parser('bdev_aio_delete', help='Delete an aio disk')
499    p.add_argument('name', help='aio bdev name')
500    p.set_defaults(func=bdev_aio_delete)
501
502    def bdev_uring_create(args):
503        print_json(rpc.bdev.bdev_uring_create(args.client,
504                                              filename=args.filename,
505                                              name=args.name,
506                                              block_size=args.block_size))
507
508    p = subparsers.add_parser('bdev_uring_create', help='Create a bdev with io_uring backend')
509    p.add_argument('filename', help='Path to device or file (ex: /dev/nvme0n1)')
510    p.add_argument('name', help='bdev name')
511    p.add_argument('block_size', help='Block size for this bdev', type=int, nargs='?')
512    p.set_defaults(func=bdev_uring_create)
513
514    def bdev_uring_delete(args):
515        rpc.bdev.bdev_uring_delete(args.client,
516                                   name=args.name)
517
518    p = subparsers.add_parser('bdev_uring_delete', help='Delete a uring bdev')
519    p.add_argument('name', help='uring bdev name')
520    p.set_defaults(func=bdev_uring_delete)
521
522    def bdev_xnvme_create(args):
523        print_json(rpc.bdev.bdev_xnvme_create(args.client,
524                                              filename=args.filename,
525                                              name=args.name,
526                                              io_mechanism=args.io_mechanism))
527
528    p = subparsers.add_parser('bdev_xnvme_create', help='Create a bdev with xNVMe backend')
529    p.add_argument('filename', help='Path to device or file (ex: /dev/nvme0n1)')
530    p.add_argument('name', help='name of xNVMe bdev to create')
531    p.add_argument('io_mechanism', help='IO mechanism to use (ex: libaio, io_uring, io_uring_cmd, etc.)')
532    p.add_argument('conserve_cpu', action='store_true', help='Whether or not to conserve CPU when polling')
533    p.set_defaults(func=bdev_xnvme_create)
534
535    def bdev_xnvme_delete(args):
536        rpc.bdev.bdev_xnvme_delete(args.client,
537                                   name=args.name)
538
539    p = subparsers.add_parser('bdev_xnvme_delete', help='Delete a xNVMe bdev')
540    p.add_argument('name', help='xNVMe bdev name')
541    p.set_defaults(func=bdev_xnvme_delete)
542
543    def bdev_nvme_set_options(args):
544        rpc.bdev.bdev_nvme_set_options(args.client,
545                                       action_on_timeout=args.action_on_timeout,
546                                       timeout_us=args.timeout_us,
547                                       timeout_admin_us=args.timeout_admin_us,
548                                       keep_alive_timeout_ms=args.keep_alive_timeout_ms,
549                                       retry_count=args.retry_count,
550                                       arbitration_burst=args.arbitration_burst,
551                                       low_priority_weight=args.low_priority_weight,
552                                       medium_priority_weight=args.medium_priority_weight,
553                                       high_priority_weight=args.high_priority_weight,
554                                       nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us,
555                                       nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us,
556                                       io_queue_requests=args.io_queue_requests,
557                                       delay_cmd_submit=args.delay_cmd_submit,
558                                       transport_retry_count=args.transport_retry_count,
559                                       bdev_retry_count=args.bdev_retry_count,
560                                       transport_ack_timeout=args.transport_ack_timeout,
561                                       ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
562                                       reconnect_delay_sec=args.reconnect_delay_sec,
563                                       fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec,
564                                       disable_auto_failback=args.disable_auto_failback,
565                                       generate_uuids=args.generate_uuids,
566                                       transport_tos=args.transport_tos,
567                                       nvme_error_stat=args.nvme_error_stat,
568                                       rdma_srq_size=args.rdma_srq_size,
569                                       io_path_stat=args.io_path_stat)
570
571    p = subparsers.add_parser('bdev_nvme_set_options',
572                              help='Set options for the bdev nvme type. This is startup command.')
573    p.add_argument('-a', '--action-on-timeout',
574                   help="Action to take on command time out. Valid values are: none, reset, abort")
575    p.add_argument('-t', '--timeout-us',
576                   help="Timeout for each command, in microseconds. If 0, don't track timeouts.", type=int)
577    p.add_argument('--timeout-admin-us',
578                   help="Timeout for each admin command, in microseconds. If 0, treat same as io timeouts.", type=int)
579    p.add_argument('-k', '--keep-alive-timeout-ms',
580                   help="Keep alive timeout period in millisecond. If 0, disable keep-alive.", type=int)
581    p.add_argument('-n', '--retry-count',
582                   help='the number of attempts per I/O when an I/O fails. (deprecated, please use --transport-retry-count.)', type=int)
583    p.add_argument('--arbitration-burst',
584                   help='the value is expressed as a power of two', type=int)
585    p.add_argument('--low-priority-weight',
586                   help='the maximum number of commands that the controller may launch at one time from a low priority queue', type=int)
587    p.add_argument('--medium-priority-weight',
588                   help='the maximum number of commands that the controller may launch at one time from a medium priority queue', type=int)
589    p.add_argument('--high-priority-weight',
590                   help='the maximum number of commands that the controller may launch at one time from a high priority queue', type=int)
591    p.add_argument('-p', '--nvme-adminq-poll-period-us',
592                   help='How often the admin queue is polled for asynchronous events', type=int)
593    p.add_argument('-i', '--nvme-ioq-poll-period-us',
594                   help='How often to poll I/O queues for completions', type=int)
595    p.add_argument('-s', '--io-queue-requests',
596                   help='The number of requests allocated for each NVMe I/O queue. Default: 512', type=int)
597    p.add_argument('-d', '--disable-delay-cmd-submit',
598                   help='Disable delaying NVMe command submission, i.e. no batching of multiple commands',
599                   action='store_false', dest='delay_cmd_submit')
600    p.add_argument('-c', '--transport-retry-count',
601                   help='the number of attempts per I/O in the transport layer when an I/O fails.', type=int)
602    p.add_argument('-r', '--bdev-retry-count',
603                   help='the number of attempts per I/O in the bdev layer when an I/O fails. -1 means infinite retries.', type=int)
604    p.add_argument('-e', '--transport-ack-timeout',
605                   help="""Time to wait ack until packet retransmission for RDMA or until closes connection for TCP.
606                   Range 0-31 where 0 is driver-specific default value.""", type=int)
607    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
608                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
609                   -1 means infinite reconnect retries. 0 means no reconnect retry.
610                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
611                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
612                   reconnect_delay_sec.
613                   This can be overridden by bdev_nvme_attach_controller.""",
614                   type=int)
615    p.add_argument('-o', '--reconnect-delay-sec',
616                   help="""Time to delay a reconnect retry.
617                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
618                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
619                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
620                   less than ctrlr_loss_timeout_sec.
621                   This can be overridden by bdev_nvme_attach_controller.""",
622                   type=int)
623    p.add_argument('-u', '--fast-io-fail-timeout-sec',
624                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
625                   0 means no such timeout.
626                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
627                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.
628                   This can be overridden by bdev_nvme_attach_controller.""",
629                   type=int)
630    p.add_argument('-f', '--disable-auto-failback',
631                   help="""Disable automatic failback. bdev_nvme_set_preferred_path can be used to do manual failback.
632                   By default, immediately failback to the preferred I/O path if it restored.""",
633                   action='store_true')
634    p.add_argument('--generate-uuids',
635                   help="""Enable generation of unique identifiers for NVMe bdevs only if they do
636                   not provide UUID themselves. These strings are based on device serial number and
637                   namespace ID and will always be the same for that device.""", action='store_true')
638    p.add_argument('--transport-tos',
639                   help="""IPv4 Type of Service value. Only applicable for RDMA transports.
640                   The default is 0 which means no TOS is applied.""", type=int)
641    p.add_argument('-m', '--nvme-error-stat', help="Enable collecting NVMe error counts.", action='store_true')
642    p.add_argument('-q', '--rdma-srq-size',
643                   help='Set the size of a shared rdma receive queue. Default: 0 (disabled)', type=int)
644    p.add_argument('--io-path-stat',
645                   help="""Enable collecting I/O path stat of each io path.""",
646                   action='store_true')
647
648    p.set_defaults(func=bdev_nvme_set_options)
649
650    def bdev_nvme_set_hotplug(args):
651        rpc.bdev.bdev_nvme_set_hotplug(args.client, enable=args.enable, period_us=args.period_us)
652
653    p = subparsers.add_parser('bdev_nvme_set_hotplug', help='Set hotplug options for bdev nvme type.')
654    p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)")
655    p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug")
656    p.add_argument('-r', '--period-us',
657                   help='How often the hotplug is processed for insert and remove events', type=int)
658    p.set_defaults(func=bdev_nvme_set_hotplug)
659
660    def bdev_nvme_attach_controller(args):
661        print_array(rpc.bdev.bdev_nvme_attach_controller(args.client,
662                                                         name=args.name,
663                                                         trtype=args.trtype,
664                                                         traddr=args.traddr,
665                                                         adrfam=args.adrfam,
666                                                         trsvcid=args.trsvcid,
667                                                         priority=args.priority,
668                                                         subnqn=args.subnqn,
669                                                         hostnqn=args.hostnqn,
670                                                         hostaddr=args.hostaddr,
671                                                         hostsvcid=args.hostsvcid,
672                                                         prchk_reftag=args.prchk_reftag,
673                                                         prchk_guard=args.prchk_guard,
674                                                         hdgst=args.hdgst,
675                                                         ddgst=args.ddgst,
676                                                         fabrics_timeout=args.fabrics_timeout,
677                                                         multipath=args.multipath,
678                                                         num_io_queues=args.num_io_queues,
679                                                         ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
680                                                         reconnect_delay_sec=args.reconnect_delay_sec,
681                                                         fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec,
682                                                         psk=args.psk,
683                                                         max_bdevs=args.max_bdevs))
684
685    p = subparsers.add_parser('bdev_nvme_attach_controller', help='Add bdevs with nvme backend')
686    p.add_argument('-b', '--name', help="Name of the NVMe controller, prefix for each bdev name", required=True)
687    p.add_argument('-t', '--trtype',
688                   help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
689    p.add_argument('-a', '--traddr',
690                   help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
691    p.add_argument('-f', '--adrfam',
692                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
693    p.add_argument('-s', '--trsvcid',
694                   help='NVMe-oF target trsvcid: e.g., a port number')
695    p.add_argument('-p', '--priority',
696                   help='NVMe-oF connection priority: e.g., a priority number')
697    p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
698    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
699    p.add_argument('-i', '--hostaddr',
700                   help='NVMe-oF host address: e.g., an ip address')
701    p.add_argument('-c', '--hostsvcid',
702                   help='NVMe-oF host svcid: e.g., a port number')
703    p.add_argument('-r', '--prchk-reftag',
704                   help='Enable checking of PI reference tag for I/O processing.', action='store_true')
705    p.add_argument('-g', '--prchk-guard',
706                   help='Enable checking of PI guard for I/O processing.', action='store_true')
707    p.add_argument('-e', '--hdgst',
708                   help='Enable TCP header digest.', action='store_true')
709    p.add_argument('-d', '--ddgst',
710                   help='Enable TCP data digest.', action='store_true')
711    p.add_argument('--fabrics-timeout', type=int, help='Fabrics connect timeout in microseconds')
712    p.add_argument('-x', '--multipath', help='Set multipath behavior (disable, failover, multipath)')
713    p.add_argument('--num-io-queues', type=int, help='Set the number of IO queues to request during initialization.')
714    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
715                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
716                   -1 means infinite reconnect retries. 0 means no reconnect retry.
717                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
718                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
719                   reconnect_delay_sec.""",
720                   type=int)
721    p.add_argument('-o', '--reconnect-delay-sec',
722                   help="""Time to delay a reconnect retry.
723                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
724                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
725                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
726                   less than ctrlr_loss_timeout_sec.""",
727                   type=int)
728    p.add_argument('-u', '--fast-io-fail-timeout-sec',
729                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
730                   0 means no such timeout.
731                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
732                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.""",
733                   type=int)
734    p.add_argument('-k', '--psk',
735                   help='Set PSK file path and enable TCP SSL socket implementation.')
736    p.add_argument('-m', '--max-bdevs', type=int,
737                   help='The size of the name array for newly created bdevs. Default is 128',)
738
739    p.set_defaults(func=bdev_nvme_attach_controller)
740
741    def bdev_nvme_get_controllers(args):
742        print_dict(rpc.nvme.bdev_nvme_get_controllers(args.client,
743                                                      name=args.name))
744
745    p = subparsers.add_parser(
746        'bdev_nvme_get_controllers', help='Display current NVMe controllers list or required NVMe controller')
747    p.add_argument('-n', '--name', help="Name of the NVMe controller. Example: Nvme0", required=False)
748    p.set_defaults(func=bdev_nvme_get_controllers)
749
750    def bdev_nvme_detach_controller(args):
751        rpc.bdev.bdev_nvme_detach_controller(args.client,
752                                             name=args.name,
753                                             trtype=args.trtype,
754                                             traddr=args.traddr,
755                                             adrfam=args.adrfam,
756                                             trsvcid=args.trsvcid,
757                                             subnqn=args.subnqn,
758                                             hostaddr=args.hostaddr,
759                                             hostsvcid=args.hostsvcid)
760
761    p = subparsers.add_parser('bdev_nvme_detach_controller',
762                              help='Detach an NVMe controller and delete any associated bdevs')
763    p.add_argument('name', help="Name of the controller")
764    p.add_argument('-t', '--trtype',
765                   help='NVMe-oF target trtype: e.g., rdma, pcie')
766    p.add_argument('-a', '--traddr',
767                   help='NVMe-oF target address: e.g., an ip address or BDF')
768    p.add_argument('-f', '--adrfam',
769                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
770    p.add_argument('-s', '--trsvcid',
771                   help='NVMe-oF target trsvcid: e.g., a port number')
772    p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
773    p.add_argument('-i', '--hostaddr',
774                   help='NVMe-oF host address: e.g., an ip address')
775    p.add_argument('-c', '--hostsvcid',
776                   help='NVMe-oF host svcid: e.g., a port number')
777    p.set_defaults(func=bdev_nvme_detach_controller)
778
779    def bdev_nvme_reset_controller(args):
780        rpc.bdev.bdev_nvme_reset_controller(args.client,
781                                            name=args.name,
782                                            cntlid=args.cntlid)
783
784    p = subparsers.add_parser('bdev_nvme_reset_controller',
785                              help='Reset an NVMe controller or all NVMe controllers in an NVMe bdev controller')
786    p.add_argument('name', help="Name of the NVMe controller")
787    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
788    p.set_defaults(func=bdev_nvme_reset_controller)
789
790    def bdev_nvme_enable_controller(args):
791        rpc.bdev.bdev_nvme_enable_controller(args.client,
792                                             name=args.name,
793                                             cntlid=args.cntlid)
794
795    p = subparsers.add_parser('bdev_nvme_enable_controller',
796                              help='Enable an NVMe controller or all NVMe controllers in an NVMe bdev controller')
797    p.add_argument('name', help="Name of the NVMe controller")
798    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
799    p.set_defaults(func=bdev_nvme_enable_controller)
800
801    def bdev_nvme_disable_controller(args):
802        rpc.bdev.bdev_nvme_disable_controller(args.client,
803                                              name=args.name,
804                                              cntlid=args.cntlid)
805
806    p = subparsers.add_parser('bdev_nvme_disable_controller',
807                              help='Disable an NVMe controller or all NVMe controllers in an NVMe bdev controller')
808    p.add_argument('name', help="Name of the NVMe controller")
809    p.add_argument('-c', '--cntlid', help="NVMe controller ID", type=int)
810    p.set_defaults(func=bdev_nvme_disable_controller)
811
812    def bdev_nvme_start_discovery(args):
813        rpc.bdev.bdev_nvme_start_discovery(args.client,
814                                           name=args.name,
815                                           trtype=args.trtype,
816                                           traddr=args.traddr,
817                                           adrfam=args.adrfam,
818                                           trsvcid=args.trsvcid,
819                                           hostnqn=args.hostnqn,
820                                           wait_for_attach=args.wait_for_attach,
821                                           attach_timeout_ms=args.attach_timeout_ms,
822                                           ctrlr_loss_timeout_sec=args.ctrlr_loss_timeout_sec,
823                                           reconnect_delay_sec=args.reconnect_delay_sec,
824                                           fast_io_fail_timeout_sec=args.fast_io_fail_timeout_sec)
825
826    p = subparsers.add_parser('bdev_nvme_start_discovery', help='Start automatic discovery')
827    p.add_argument('-b', '--name', help="Name of the NVMe controller prefix for each bdev name", required=True)
828    p.add_argument('-t', '--trtype',
829                   help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
830    p.add_argument('-a', '--traddr',
831                   help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
832    p.add_argument('-f', '--adrfam',
833                   help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
834    p.add_argument('-s', '--trsvcid',
835                   help='NVMe-oF target trsvcid: e.g., a port number')
836    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
837    p.add_argument('-w', '--wait-for-attach', action='store_true',
838                   help='Do not complete RPC until all discovered NVM subsystems are attached')
839    p.add_argument('-T', '--attach-timeout-ms', type=int, required=False,
840                   help="""Time to wait until the discovery and all discovered NVM subsystems
841                        are attached (default: 0, meaning wait indefinitely).  Automatically
842                        selects the --wait-for-attach option.""")
843    p.add_argument('-l', '--ctrlr-loss-timeout-sec',
844                   help="""Time to wait until ctrlr is reconnected before deleting ctrlr.
845                   -1 means infinite reconnect retries. 0 means no reconnect retry.
846                   If reconnect_delay_sec is zero, ctrlr_loss_timeout_sec has to be zero.
847                   If reconnect_delay_sec is non-zero, ctrlr_loss_timeout_sec has to be -1 or not less than
848                   reconnect_delay_sec.""",
849                   type=int)
850    p.add_argument('-o', '--reconnect-delay-sec',
851                   help="""Time to delay a reconnect retry.
852                   If ctrlr_loss_timeout_sec is zero, reconnect_delay_sec has to be zero.
853                   If ctrlr_loss_timeout_sec is -1, reconnect_delay_sec has to be non-zero.
854                   If ctrlr_loss_timeout_sec is not -1 or zero, reconnect_delay_sec has to be non-zero and
855                   less than ctrlr_loss_timeout_sec.""",
856                   type=int)
857    p.add_argument('-u', '--fast-io-fail-timeout-sec',
858                   help="""Time to wait until ctrlr is reconnected before failing I/O to ctrlr.
859                   0 means no such timeout.
860                   If fast_io_fail_timeout_sec is not zero, it has to be not less than reconnect_delay_sec and
861                   less than ctrlr_loss_timeout_sec if ctrlr_loss_timeout_sec is not -1.""",
862                   type=int)
863    p.set_defaults(func=bdev_nvme_start_discovery)
864
865    def bdev_nvme_stop_discovery(args):
866        rpc.bdev.bdev_nvme_stop_discovery(args.client, name=args.name)
867
868    p = subparsers.add_parser('bdev_nvme_stop_discovery', help='Stop automatic discovery')
869    p.add_argument('-b', '--name', help="Name of the service to stop", required=True)
870    p.set_defaults(func=bdev_nvme_stop_discovery)
871
872    def bdev_nvme_get_discovery_info(args):
873        print_dict(rpc.bdev.bdev_nvme_get_discovery_info(args.client))
874
875    p = subparsers.add_parser('bdev_nvme_get_discovery_info', help='Get information about the automatic discovery')
876    p.set_defaults(func=bdev_nvme_get_discovery_info)
877
878    def bdev_nvme_get_io_paths(args):
879        print_dict(rpc.bdev.bdev_nvme_get_io_paths(args.client, name=args.name))
880
881    p = subparsers.add_parser('bdev_nvme_get_io_paths', help='Display active I/O paths')
882    p.add_argument('-n', '--name', help="Name of the NVMe bdev", required=False)
883    p.set_defaults(func=bdev_nvme_get_io_paths)
884
885    def bdev_nvme_set_preferred_path(args):
886        rpc.bdev.bdev_nvme_set_preferred_path(args.client,
887                                              name=args.name,
888                                              cntlid=args.cntlid)
889
890    p = subparsers.add_parser('bdev_nvme_set_preferred_path',
891                              help="""Set the preferred I/O path for an NVMe bdev when in multipath mode""")
892    p.add_argument('-b', '--name', help='Name of the NVMe bdev', required=True)
893    p.add_argument('-c', '--cntlid', help='NVMe-oF controller ID', type=int, required=True)
894    p.set_defaults(func=bdev_nvme_set_preferred_path)
895
896    def bdev_nvme_set_multipath_policy(args):
897        rpc.bdev.bdev_nvme_set_multipath_policy(args.client,
898                                                name=args.name,
899                                                policy=args.policy,
900                                                selector=args.selector,
901                                                rr_min_io=args.rr_min_io)
902
903    p = subparsers.add_parser('bdev_nvme_set_multipath_policy',
904                              help="""Set multipath policy of the NVMe bdev""")
905    p.add_argument('-b', '--name', help='Name of the NVMe bdev', required=True)
906    p.add_argument('-p', '--policy', help='Multipath policy (active_passive or active_active)', required=True)
907    p.add_argument('-s', '--selector', help='Multipath selector (round_robin, queue_depth)', required=False)
908    p.add_argument('-r', '--rr-min-io',
909                   help='Number of IO to route to a path before switching to another for round-robin',
910                   type=int, required=False)
911    p.set_defaults(func=bdev_nvme_set_multipath_policy)
912
913    def bdev_nvme_get_path_iostat(args):
914        print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client,
915                                                      name=args.name))
916
917    p = subparsers.add_parser('bdev_nvme_get_path_iostat',
918                              help="""Display current I/O statistics of all the IO paths of the blockdev. It can be
919                              called when io_path_stat is true.""")
920    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True)
921    p.set_defaults(func=bdev_nvme_get_path_iostat)
922
923    def bdev_nvme_cuse_register(args):
924        rpc.bdev.bdev_nvme_cuse_register(args.client,
925                                         name=args.name)
926
927    p = subparsers.add_parser('bdev_nvme_cuse_register',
928                              help='Register CUSE devices on NVMe controller')
929    p.add_argument('-n', '--name',
930                   help='Name of the NVMe controller. Example: Nvme0', required=True)
931    p.set_defaults(func=bdev_nvme_cuse_register)
932
933    def bdev_nvme_cuse_unregister(args):
934        rpc.bdev.bdev_nvme_cuse_unregister(args.client,
935                                           name=args.name)
936
937    p = subparsers.add_parser('bdev_nvme_cuse_unregister',
938                              help='Unregister CUSE devices on NVMe controller')
939    p.add_argument('-n', '--name',
940                   help='Name of the NVMe controller. Example: Nvme0', required=True)
941    p.set_defaults(func=bdev_nvme_cuse_unregister)
942
943    def bdev_zone_block_create(args):
944        print_json(rpc.bdev.bdev_zone_block_create(args.client,
945                                                   name=args.name,
946                                                   base_bdev=args.base_bdev,
947                                                   zone_capacity=args.zone_capacity,
948                                                   optimal_open_zones=args.optimal_open_zones))
949
950    p = subparsers.add_parser('bdev_zone_block_create',
951                              help='Create virtual zone namespace device with block device backend')
952    p.add_argument('-b', '--name', help="Name of the zone device", required=True)
953    p.add_argument('-n', '--base-bdev', help='Name of underlying, non-zoned bdev', required=True)
954    p.add_argument('-z', '--zone-capacity', help='Surfaced zone capacity in blocks', type=int, required=True)
955    p.add_argument('-o', '--optimal-open-zones', help='Number of zones required to reach optimal write speed', type=int, required=True)
956    p.set_defaults(func=bdev_zone_block_create)
957
958    def bdev_zone_block_delete(args):
959        rpc.bdev.bdev_zone_block_delete(args.client,
960                                        name=args.name)
961
962    p = subparsers.add_parser('bdev_zone_block_delete', help='Delete a virtual zone namespace device')
963    p.add_argument('name', help='Virtual zone bdev name')
964    p.set_defaults(func=bdev_zone_block_delete)
965
966    def bdev_rbd_register_cluster(args):
967        config_param = None
968        if args.config_param:
969            config_param = {}
970            for entry in args.config_param:
971                parts = entry.split('=', 1)
972                if len(parts) != 2:
973                    raise Exception('--config %s not in key=value form' % entry)
974                config_param[parts[0]] = parts[1]
975        print_json(rpc.bdev.bdev_rbd_register_cluster(args.client,
976                                                      name=args.name,
977                                                      user=args.user,
978                                                      config_param=config_param,
979                                                      config_file=args.config_file,
980                                                      key_file=args.key_file,
981                                                      core_mask=args.core_mask))
982
983    p = subparsers.add_parser('bdev_rbd_register_cluster',
984                              help='Add a Rados cluster with ceph rbd backend')
985    p.add_argument('name', help="Name of the Rados cluster only known to rbd bdev")
986    p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)", required=False)
987    p.add_argument('--config-param', action='append', metavar='key=value',
988                   help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
989    p.add_argument('--config-file', help="The file path of the Rados configuration file", required=False)
990    p.add_argument('--key-file', help="The file path of the Rados keyring file", required=False)
991    p.add_argument('--core-mask', help="Set core mask for librbd IO context threads", required=False)
992    p.set_defaults(func=bdev_rbd_register_cluster)
993
994    def bdev_rbd_unregister_cluster(args):
995        rpc.bdev.bdev_rbd_unregister_cluster(args.client, name=args.name)
996
997    p = subparsers.add_parser('bdev_rbd_unregister_cluster',
998                              help='Unregister a Rados cluster object')
999    p.add_argument('name', help='Name of the Rados Cluster only known to rbd bdev')
1000    p.set_defaults(func=bdev_rbd_unregister_cluster)
1001
1002    def bdev_rbd_get_clusters_info(args):
1003        print_json(rpc.bdev.bdev_rbd_get_clusters_info(args.client, name=args.name))
1004
1005    p = subparsers.add_parser('bdev_rbd_get_clusters_info',
1006                              help='Display registered Rados Cluster names and related info')
1007    p.add_argument('-b', '--name', help="Name of the registered Rados Cluster Name. Example: Cluster1", required=False)
1008    p.set_defaults(func=bdev_rbd_get_clusters_info)
1009
1010    def bdev_rbd_create(args):
1011        config = None
1012        if args.config:
1013            config = {}
1014            for entry in args.config:
1015                parts = entry.split('=', 1)
1016                if len(parts) != 2:
1017                    raise Exception('--config %s not in key=value form' % entry)
1018                config[parts[0]] = parts[1]
1019        print_json(rpc.bdev.bdev_rbd_create(args.client,
1020                                            name=args.name,
1021                                            user=args.user,
1022                                            config=config,
1023                                            pool_name=args.pool_name,
1024                                            rbd_name=args.rbd_name,
1025                                            block_size=args.block_size,
1026                                            cluster_name=args.cluster_name,
1027                                            uuid=args.uuid))
1028
1029    p = subparsers.add_parser('bdev_rbd_create', help='Add a bdev with ceph rbd backend')
1030    p.add_argument('-b', '--name', help="Name of the bdev", required=False)
1031    p.add_argument('--user', help="Ceph user name (i.e. admin, not client.admin)", required=False)
1032    p.add_argument('--config', action='append', metavar='key=value',
1033                   help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
1034    p.add_argument('pool_name', help='rbd pool name')
1035    p.add_argument('rbd_name', help='rbd image name')
1036    p.add_argument('block_size', help='rbd block size', type=int)
1037    p.add_argument('-c', '--cluster-name', help="cluster name to identify the Rados cluster", required=False)
1038    p.add_argument('-u', '--uuid', help="UUID of the bdev")
1039    p.set_defaults(func=bdev_rbd_create)
1040
1041    def bdev_rbd_delete(args):
1042        rpc.bdev.bdev_rbd_delete(args.client,
1043                                 name=args.name)
1044
1045    p = subparsers.add_parser('bdev_rbd_delete', help='Delete a rbd bdev')
1046    p.add_argument('name', help='rbd bdev name')
1047    p.set_defaults(func=bdev_rbd_delete)
1048
1049    def bdev_rbd_resize(args):
1050        print_json(rpc.bdev.bdev_rbd_resize(args.client,
1051                                            name=args.name,
1052                                            new_size=int(args.new_size)))
1053
1054    p = subparsers.add_parser('bdev_rbd_resize',
1055                              help='Resize a rbd bdev')
1056    p.add_argument('name', help='rbd bdev name')
1057    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
1058    p.set_defaults(func=bdev_rbd_resize)
1059
1060    def bdev_delay_create(args):
1061        print_json(rpc.bdev.bdev_delay_create(args.client,
1062                                              base_bdev_name=args.base_bdev_name,
1063                                              name=args.name,
1064                                              uuid=args.uuid,
1065                                              avg_read_latency=args.avg_read_latency,
1066                                              p99_read_latency=args.nine_nine_read_latency,
1067                                              avg_write_latency=args.avg_write_latency,
1068                                              p99_write_latency=args.nine_nine_write_latency))
1069
1070    p = subparsers.add_parser('bdev_delay_create',
1071                              help='Add a delay bdev on existing bdev')
1072    p.add_argument('-b', '--base-bdev-name', help="Name of the existing bdev", required=True)
1073    p.add_argument('-d', '--name', help="Name of the delay bdev", required=True)
1074    p.add_argument('-u', '--uuid', help='UUID of the bdev (optional)')
1075    p.add_argument('-r', '--avg-read-latency',
1076                   help="Average latency to apply before completing read ops (in microseconds)", required=True, type=int)
1077    p.add_argument('-t', '--nine-nine-read-latency',
1078                   help="latency to apply to 1 in 100 read ops (in microseconds)", required=True, type=int)
1079    p.add_argument('-w', '--avg-write-latency',
1080                   help="Average latency to apply before completing write ops (in microseconds)", required=True, type=int)
1081    p.add_argument('-n', '--nine-nine-write-latency',
1082                   help="latency to apply to 1 in 100 write ops (in microseconds)", required=True, type=int)
1083    p.set_defaults(func=bdev_delay_create)
1084
1085    def bdev_delay_delete(args):
1086        rpc.bdev.bdev_delay_delete(args.client,
1087                                   name=args.name)
1088
1089    p = subparsers.add_parser('bdev_delay_delete', help='Delete a delay bdev')
1090    p.add_argument('name', help='delay bdev name')
1091    p.set_defaults(func=bdev_delay_delete)
1092
1093    def bdev_delay_update_latency(args):
1094        print_json(rpc.bdev.bdev_delay_update_latency(args.client,
1095                                                      delay_bdev_name=args.delay_bdev_name,
1096                                                      latency_type=args.latency_type,
1097                                                      latency_us=args.latency_us))
1098    p = subparsers.add_parser('bdev_delay_update_latency',
1099                              help='Update one of the latency values for a given delay bdev')
1100    p.add_argument('delay_bdev_name', help='The name of the given delay bdev')
1101    p.add_argument('latency_type', help='one of: avg_read, avg_write, p99_read, p99_write. No other values accepted.')
1102    p.add_argument('latency_us', help='new latency value in microseconds.', type=int)
1103    p.set_defaults(func=bdev_delay_update_latency)
1104
1105    def bdev_error_create(args):
1106        print_json(rpc.bdev.bdev_error_create(args.client,
1107                                              base_name=args.base_name,
1108                                              uuid=args.uuid))
1109
1110    p = subparsers.add_parser('bdev_error_create', help='Add bdev with error injection backend')
1111    p.add_argument('base_name', help='base bdev name')
1112    p.add_argument('--uuid', help='UUID for this bdev', required=False)
1113    p.set_defaults(func=bdev_error_create)
1114
1115    def bdev_error_delete(args):
1116        rpc.bdev.bdev_error_delete(args.client,
1117                                   name=args.name)
1118
1119    p = subparsers.add_parser('bdev_error_delete', help='Delete an error bdev')
1120    p.add_argument('name', help='error bdev name')
1121    p.set_defaults(func=bdev_error_delete)
1122
1123    def bdev_iscsi_set_options(args):
1124        rpc.bdev.bdev_iscsi_set_options(args.client,
1125                                        timeout_sec=args.timeout_sec)
1126
1127    p = subparsers.add_parser('bdev_iscsi_set_options', help='Set options for the bdev iscsi type.')
1128    p.add_argument('-t', '--timeout-sec', help="Timeout for command, in seconds, if 0, don't track timeout.", type=int)
1129    p.set_defaults(func=bdev_iscsi_set_options)
1130
1131    def bdev_iscsi_create(args):
1132        print_json(rpc.bdev.bdev_iscsi_create(args.client,
1133                                              name=args.name,
1134                                              url=args.url,
1135                                              initiator_iqn=args.initiator_iqn))
1136
1137    p = subparsers.add_parser('bdev_iscsi_create',
1138                              help='Add bdev with iSCSI initiator backend')
1139    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
1140    p.add_argument('-i', '--initiator-iqn', help="Initiator IQN", required=True)
1141    p.add_argument('--url', help="iSCSI Lun URL", required=True)
1142    p.set_defaults(func=bdev_iscsi_create)
1143
1144    def bdev_iscsi_delete(args):
1145        rpc.bdev.bdev_iscsi_delete(args.client,
1146                                   name=args.name)
1147
1148    p = subparsers.add_parser('bdev_iscsi_delete', help='Delete an iSCSI bdev')
1149    p.add_argument('name', help='iSCSI bdev name')
1150    p.set_defaults(func=bdev_iscsi_delete)
1151
1152    def bdev_passthru_create(args):
1153        print_json(rpc.bdev.bdev_passthru_create(args.client,
1154                                                 base_bdev_name=args.base_bdev_name,
1155                                                 name=args.name))
1156
1157    p = subparsers.add_parser('bdev_passthru_create', help='Add a pass through bdev on existing bdev')
1158    p.add_argument('-b', '--base-bdev-name', help="Name of the existing bdev", required=True)
1159    p.add_argument('-p', '--name', help="Name of the pass through bdev", required=True)
1160    p.set_defaults(func=bdev_passthru_create)
1161
1162    def bdev_passthru_delete(args):
1163        rpc.bdev.bdev_passthru_delete(args.client,
1164                                      name=args.name)
1165
1166    p = subparsers.add_parser('bdev_passthru_delete', help='Delete a pass through bdev')
1167    p.add_argument('name', help='pass through bdev name')
1168    p.set_defaults(func=bdev_passthru_delete)
1169
1170    def bdev_get_bdevs(args):
1171        print_dict(rpc.bdev.bdev_get_bdevs(args.client,
1172                                           name=args.name, timeout=args.timeout_ms))
1173
1174    p = subparsers.add_parser('bdev_get_bdevs',
1175                              help='Display current blockdev list or required blockdev')
1176    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
1177    p.add_argument('-t', '--timeout-ms', help="""Time in ms to wait for the bdev to appear (only used
1178    with the -b|--name option). The default timeout is 0, meaning the RPC returns immediately
1179    whether the bdev exists or not.""",
1180                   type=int, required=False)
1181    p.set_defaults(func=bdev_get_bdevs)
1182
1183    def bdev_get_iostat(args):
1184        print_dict(rpc.bdev.bdev_get_iostat(args.client,
1185                                            name=args.name,
1186                                            per_channel=args.per_channel))
1187
1188    p = subparsers.add_parser('bdev_get_iostat',
1189                              help='Display current I/O statistics of all the blockdevs or specified blockdev.')
1190    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
1191    p.add_argument('-c', '--per-channel', default=False, dest='per_channel', help='Display per channel IO stats for specified device',
1192                   action='store_true', required=False)
1193    p.set_defaults(func=bdev_get_iostat)
1194
1195    def bdev_reset_iostat(args):
1196        rpc.bdev.bdev_reset_iostat(args.client, name=args.name, mode=args.mode)
1197
1198    p = subparsers.add_parser('bdev_reset_iostat',
1199                              help='Reset I/O statistics of all the blockdevs or specified blockdev.')
1200    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
1201    p.add_argument('-m', '--mode', help="Mode to reset I/O statistics", choices=['all', 'maxmin'], required=False)
1202    p.set_defaults(func=bdev_reset_iostat)
1203
1204    def bdev_enable_histogram(args):
1205        rpc.bdev.bdev_enable_histogram(args.client, name=args.name, enable=args.enable)
1206
1207    p = subparsers.add_parser('bdev_enable_histogram',
1208                              help='Enable or disable histogram for specified bdev')
1209    p.add_argument('-e', '--enable', default=True, dest='enable', action='store_true', help='Enable histograms on specified device')
1210    p.add_argument('-d', '--disable', dest='enable', action='store_false', help='Disable histograms on specified device')
1211    p.add_argument('name', help='bdev name')
1212    p.set_defaults(func=bdev_enable_histogram)
1213
1214    def bdev_get_histogram(args):
1215        print_dict(rpc.bdev.bdev_get_histogram(args.client, name=args.name))
1216
1217    p = subparsers.add_parser('bdev_get_histogram',
1218                              help='Get histogram for specified bdev')
1219    p.add_argument('name', help='bdev name')
1220    p.set_defaults(func=bdev_get_histogram)
1221
1222    def bdev_set_qd_sampling_period(args):
1223        rpc.bdev.bdev_set_qd_sampling_period(args.client,
1224                                             name=args.name,
1225                                             period=args.period)
1226
1227    p = subparsers.add_parser('bdev_set_qd_sampling_period',
1228                              help="Enable or disable tracking of a bdev's queue depth.")
1229    p.add_argument('name', help='Blockdev name. Example: Malloc0')
1230    p.add_argument('period', help='Period with which to poll the block device queue depth in microseconds.'
1231                   ' If set to 0, polling will be disabled.',
1232                   type=int)
1233    p.set_defaults(func=bdev_set_qd_sampling_period)
1234
1235    def bdev_set_qos_limit(args):
1236        rpc.bdev.bdev_set_qos_limit(args.client,
1237                                    name=args.name,
1238                                    rw_ios_per_sec=args.rw_ios_per_sec,
1239                                    rw_mbytes_per_sec=args.rw_mbytes_per_sec,
1240                                    r_mbytes_per_sec=args.r_mbytes_per_sec,
1241                                    w_mbytes_per_sec=args.w_mbytes_per_sec)
1242
1243    p = subparsers.add_parser('bdev_set_qos_limit',
1244                              help='Set QoS rate limit on a blockdev')
1245    p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0')
1246    p.add_argument('--rw-ios-per-sec',
1247                   help='R/W IOs per second limit (>=1000, example: 20000). 0 means unlimited.',
1248                   type=int, required=False)
1249    p.add_argument('--rw-mbytes-per-sec',
1250                   help="R/W megabytes per second limit (>=10, example: 100). 0 means unlimited.",
1251                   type=int, required=False)
1252    p.add_argument('--r-mbytes-per-sec',
1253                   help="Read megabytes per second limit (>=10, example: 100). 0 means unlimited.",
1254                   type=int, required=False)
1255    p.add_argument('--w-mbytes-per-sec',
1256                   help="Write megabytes per second limit (>=10, example: 100). 0 means unlimited.",
1257                   type=int, required=False)
1258    p.set_defaults(func=bdev_set_qos_limit)
1259
1260    def bdev_error_inject_error(args):
1261        rpc.bdev.bdev_error_inject_error(args.client,
1262                                         name=args.name,
1263                                         io_type=args.io_type,
1264                                         error_type=args.error_type,
1265                                         num=args.num,
1266                                         corrupt_offset=args.corrupt_offset,
1267                                         corrupt_value=args.corrupt_value)
1268
1269    p = subparsers.add_parser('bdev_error_inject_error', help='bdev inject error')
1270    p.add_argument('name', help="""the name of the error injection bdev""")
1271    p.add_argument('io_type', help="""io_type: 'clear' 'read' 'write' 'unmap' 'flush' 'all'""")
1272    p.add_argument('error_type', help="""error_type: 'failure' 'pending' 'corrupt_data'""")
1273    p.add_argument(
1274        '-n', '--num', help='the number of commands you want to fail', type=int)
1275    p.add_argument(
1276        '-o', '--corrupt-offset', help='the offset in bytes to xor with corrupt_value', type=int)
1277    p.add_argument(
1278        '-v', '--corrupt-value', help='the value for xor (1-255, 0 is invalid)', type=int)
1279    p.set_defaults(func=bdev_error_inject_error)
1280
1281    def bdev_nvme_apply_firmware(args):
1282        print_dict(rpc.bdev.bdev_nvme_apply_firmware(args.client,
1283                                                     bdev_name=args.bdev_name,
1284                                                     filename=args.filename))
1285
1286    p = subparsers.add_parser('bdev_nvme_apply_firmware', help='Download and commit firmware to NVMe device')
1287    p.add_argument('filename', help='filename of the firmware to download')
1288    p.add_argument('bdev_name', help='name of the NVMe device')
1289    p.set_defaults(func=bdev_nvme_apply_firmware)
1290
1291    def bdev_nvme_get_transport_statistics(args):
1292        print_dict(rpc.bdev.bdev_nvme_get_transport_statistics(args.client))
1293
1294    p = subparsers.add_parser('bdev_nvme_get_transport_statistics',
1295                              help='Get bdev_nvme poll group transport statistics')
1296    p.set_defaults(func=bdev_nvme_get_transport_statistics)
1297
1298    def bdev_nvme_get_controller_health_info(args):
1299        print_dict(rpc.bdev.bdev_nvme_get_controller_health_info(args.client,
1300                                                                 name=args.name))
1301
1302    p = subparsers.add_parser('bdev_nvme_get_controller_health_info',
1303                              help='Display health log of the required NVMe bdev controller.')
1304    p.add_argument('-c', '--name', help="Name of the NVMe bdev controller. Example: Nvme0", required=True)
1305    p.set_defaults(func=bdev_nvme_get_controller_health_info)
1306
1307    # iSCSI
1308    def iscsi_set_options(args):
1309        rpc.iscsi.iscsi_set_options(
1310            args.client,
1311            auth_file=args.auth_file,
1312            node_base=args.node_base,
1313            nop_timeout=args.nop_timeout,
1314            nop_in_interval=args.nop_in_interval,
1315            disable_chap=args.disable_chap,
1316            require_chap=args.require_chap,
1317            mutual_chap=args.mutual_chap,
1318            chap_group=args.chap_group,
1319            max_sessions=args.max_sessions,
1320            max_queue_depth=args.max_queue_depth,
1321            max_connections_per_session=args.max_connections_per_session,
1322            default_time2wait=args.default_time2wait,
1323            default_time2retain=args.default_time2retain,
1324            first_burst_length=args.first_burst_length,
1325            immediate_data=args.immediate_data,
1326            error_recovery_level=args.error_recovery_level,
1327            allow_duplicated_isid=args.allow_duplicated_isid,
1328            max_large_datain_per_connection=args.max_large_datain_per_connection,
1329            max_r2t_per_connection=args.max_r2t_per_connection,
1330            pdu_pool_size=args.pdu_pool_size,
1331            immediate_data_pool_size=args.immediate_data_pool_size,
1332            data_out_pool_size=args.data_out_pool_size)
1333
1334    p = subparsers.add_parser('iscsi_set_options',
1335                              help="""Set options of iSCSI subsystem""")
1336    p.add_argument('-f', '--auth-file', help='Path to CHAP shared secret file')
1337    p.add_argument('-b', '--node-base', help='Prefix of the name of iSCSI target node')
1338    p.add_argument('-o', '--nop-timeout', help='Timeout in seconds to nop-in request to the initiator', type=int)
1339    p.add_argument('-n', '--nop-in-interval', help='Time interval in secs between nop-in requests by the target', type=int)
1340    p.add_argument('-d', '--disable-chap', help="""CHAP for discovery session should be disabled.
1341    *** Mutually exclusive with --require-chap""", action='store_true')
1342    p.add_argument('-r', '--require-chap', help="""CHAP for discovery session should be required.
1343    *** Mutually exclusive with --disable-chap""", action='store_true')
1344    p.add_argument('-m', '--mutual-chap', help='CHAP for discovery session should be mutual', action='store_true')
1345    p.add_argument('-g', '--chap-group', help="""Authentication group ID for discovery session.
1346    *** Authentication group must be precreated ***""", type=int)
1347    p.add_argument('-a', '--max-sessions', help='Maximum number of sessions in the host.', type=int)
1348    p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/Os per queue.', type=int)
1349    p.add_argument('-c', '--max-connections-per-session', help='Negotiated parameter, MaxConnections.', type=int)
1350    p.add_argument('-w', '--default-time2wait', help='Negotiated parameter, DefaultTime2Wait.', type=int)
1351    p.add_argument('-v', '--default-time2retain', help='Negotiated parameter, DefaultTime2Retain.', type=int)
1352    p.add_argument('-s', '--first-burst-length', help='Negotiated parameter, FirstBurstLength.', type=int)
1353    p.add_argument('-i', '--immediate-data', help='Negotiated parameter, ImmediateData.', action='store_true')
1354    p.add_argument('-l', '--error-recovery-level', help='Negotiated parameter, ErrorRecoveryLevel', type=int)
1355    p.add_argument('-p', '--allow-duplicated-isid', help='Allow duplicated initiator session ID.', action='store_true')
1356    p.add_argument('-x', '--max-large-datain-per-connection', help='Max number of outstanding split read I/Os per connection', type=int)
1357    p.add_argument('-k', '--max-r2t-per-connection', help='Max number of outstanding R2Ts per connection', type=int)
1358    p.add_argument('-u', '--pdu-pool-size', help='Number of PDUs in the pool', type=int)
1359    p.add_argument('-j', '--immediate-data-pool-size', help='Number of immediate data buffers in the pool', type=int)
1360    p.add_argument('-z', '--data-out-pool-size', help='Number of data out buffers in the pool', type=int)
1361    p.set_defaults(func=iscsi_set_options)
1362
1363    def iscsi_set_discovery_auth(args):
1364        rpc.iscsi.iscsi_set_discovery_auth(
1365            args.client,
1366            disable_chap=args.disable_chap,
1367            require_chap=args.require_chap,
1368            mutual_chap=args.mutual_chap,
1369            chap_group=args.chap_group)
1370
1371    p = subparsers.add_parser('iscsi_set_discovery_auth',
1372                              help="""Set CHAP authentication for discovery session.""")
1373    p.add_argument('-d', '--disable-chap', help="""CHAP for discovery session should be disabled.
1374    *** Mutually exclusive with --require-chap""", action='store_true')
1375    p.add_argument('-r', '--require-chap', help="""CHAP for discovery session should be required.
1376    *** Mutually exclusive with --disable-chap""", action='store_true')
1377    p.add_argument('-m', '--mutual-chap', help='CHAP for discovery session should be mutual', action='store_true')
1378    p.add_argument('-g', '--chap-group', help="""Authentication group ID for discovery session.
1379    *** Authentication group must be precreated ***""", type=int)
1380    p.set_defaults(func=iscsi_set_discovery_auth)
1381
1382    def iscsi_create_auth_group(args):
1383        secrets = None
1384        if args.secrets:
1385            secrets = [dict(u.split(":") for u in a.split(" ")) for a in args.secrets.split(",")]
1386
1387        rpc.iscsi.iscsi_create_auth_group(args.client, tag=args.tag, secrets=secrets)
1388
1389    p = subparsers.add_parser('iscsi_create_auth_group',
1390                              help='Create authentication group for CHAP authentication.')
1391    p.add_argument('tag', help='Authentication group tag (unique, integer > 0).', type=int)
1392    p.add_argument('-c', '--secrets', help="""Comma-separated list of CHAP secrets
1393<user:user_name secret:chap_secret muser:mutual_user_name msecret:mutual_chap_secret> enclosed in quotes.
1394Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 msecret:ms2'""", required=False)
1395    p.set_defaults(func=iscsi_create_auth_group)
1396
1397    def iscsi_delete_auth_group(args):
1398        rpc.iscsi.iscsi_delete_auth_group(args.client, tag=args.tag)
1399
1400    p = subparsers.add_parser('iscsi_delete_auth_group',
1401                              help='Delete an authentication group.')
1402    p.add_argument('tag', help='Authentication group tag', type=int)
1403    p.set_defaults(func=iscsi_delete_auth_group)
1404
1405    def iscsi_auth_group_add_secret(args):
1406        rpc.iscsi.iscsi_auth_group_add_secret(
1407            args.client,
1408            tag=args.tag,
1409            user=args.user,
1410            secret=args.secret,
1411            muser=args.muser,
1412            msecret=args.msecret)
1413
1414    p = subparsers.add_parser('iscsi_auth_group_add_secret',
1415                              help='Add a secret to an authentication group.')
1416    p.add_argument('tag', help='Authentication group tag', type=int)
1417    p.add_argument('-u', '--user', help='User name for one-way CHAP authentication', required=True)
1418    p.add_argument('-s', '--secret', help='Secret for one-way CHAP authentication', required=True)
1419    p.add_argument('-m', '--muser', help='User name for mutual CHAP authentication')
1420    p.add_argument('-r', '--msecret', help='Secret for mutual CHAP authentication')
1421    p.set_defaults(func=iscsi_auth_group_add_secret)
1422
1423    def iscsi_auth_group_remove_secret(args):
1424        rpc.iscsi.iscsi_auth_group_remove_secret(args.client, tag=args.tag, user=args.user)
1425
1426    p = subparsers.add_parser('iscsi_auth_group_remove_secret',
1427                              help='Remove a secret from an authentication group.')
1428    p.add_argument('tag', help='Authentication group tag', type=int)
1429    p.add_argument('-u', '--user', help='User name for one-way CHAP authentication', required=True)
1430    p.set_defaults(func=iscsi_auth_group_remove_secret)
1431
1432    def iscsi_get_auth_groups(args):
1433        print_dict(rpc.iscsi.iscsi_get_auth_groups(args.client))
1434
1435    p = subparsers.add_parser('iscsi_get_auth_groups',
1436                              help='Display current authentication group configuration')
1437    p.set_defaults(func=iscsi_get_auth_groups)
1438
1439    def iscsi_get_portal_groups(args):
1440        print_dict(rpc.iscsi.iscsi_get_portal_groups(args.client))
1441
1442    p = subparsers.add_parser('iscsi_get_portal_groups', help='Display current portal group configuration')
1443    p.set_defaults(func=iscsi_get_portal_groups)
1444
1445    def iscsi_get_initiator_groups(args):
1446        print_dict(rpc.iscsi.iscsi_get_initiator_groups(args.client))
1447
1448    p = subparsers.add_parser('iscsi_get_initiator_groups',
1449                              help='Display current initiator group configuration')
1450    p.set_defaults(func=iscsi_get_initiator_groups)
1451
1452    def iscsi_get_target_nodes(args):
1453        print_dict(rpc.iscsi.iscsi_get_target_nodes(args.client))
1454
1455    p = subparsers.add_parser('iscsi_get_target_nodes', help='Display target nodes')
1456    p.set_defaults(func=iscsi_get_target_nodes)
1457
1458    def iscsi_create_target_node(args):
1459        luns = []
1460        for u in args.bdev_name_id_pairs.strip().split(" "):
1461            bdev_name, lun_id = u.split(":")
1462            luns.append({"bdev_name": bdev_name, "lun_id": int(lun_id)})
1463
1464        pg_ig_maps = []
1465        for u in args.pg_ig_mappings.strip().split(" "):
1466            pg, ig = u.split(":")
1467            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1468
1469        rpc.iscsi.iscsi_create_target_node(
1470            args.client,
1471            luns=luns,
1472            pg_ig_maps=pg_ig_maps,
1473            name=args.name,
1474            alias_name=args.alias_name,
1475            queue_depth=args.queue_depth,
1476            chap_group=args.chap_group,
1477            disable_chap=args.disable_chap,
1478            require_chap=args.require_chap,
1479            mutual_chap=args.mutual_chap,
1480            header_digest=args.header_digest,
1481            data_digest=args.data_digest)
1482
1483    p = subparsers.add_parser('iscsi_create_target_node', help='Add a target node')
1484    p.add_argument('name', help='Target node name (ASCII)')
1485    p.add_argument('alias_name', help='Target node alias name (ASCII)')
1486    p.add_argument('bdev_name_id_pairs', help="""Whitespace-separated list of <bdev name:LUN ID> pairs enclosed
1487    in quotes.  Format:  'bdev_name0:id0 bdev_name1:id1' etc
1488    Example: 'Malloc0:0 Malloc1:1 Malloc5:2'
1489    *** The bdevs must pre-exist ***
1490    *** LUN0 (id = 0) is required ***
1491    *** bdevs names cannot contain space or colon characters ***""")
1492    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1493    Whitespace separated, quoted, mapping defined with colon
1494    separated list of "tags" (int > 0)
1495    Example: '1:1 2:2 2:1'
1496    *** The Portal/Initiator Groups must be precreated ***""")
1497    p.add_argument('queue_depth', help='Desired target queue depth', type=int)
1498    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this target node.
1499    *** Authentication group must be precreated ***""", type=int)
1500    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this target node.
1501    *** Mutually exclusive with --require-chap ***""", action='store_true')
1502    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this target node.
1503    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1504    p.add_argument(
1505        '-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.', action='store_true')
1506    p.add_argument('-H', '--header-digest',
1507                   help='Header Digest should be required for this target node.', action='store_true')
1508    p.add_argument('-D', '--data-digest',
1509                   help='Data Digest should be required for this target node.', action='store_true')
1510    p.set_defaults(func=iscsi_create_target_node)
1511
1512    def iscsi_target_node_add_lun(args):
1513        rpc.iscsi.iscsi_target_node_add_lun(
1514            args.client,
1515            name=args.name,
1516            bdev_name=args.bdev_name,
1517            lun_id=args.lun_id)
1518
1519    p = subparsers.add_parser('iscsi_target_node_add_lun',
1520                              help='Add LUN to the target node')
1521    p.add_argument('name', help='Target node name (ASCII)')
1522    p.add_argument('bdev_name', help="""bdev name enclosed in quotes.
1523    *** bdev name cannot contain space or colon characters ***""")
1524    p.add_argument('-i', dest='lun_id', help="""LUN ID (integer >= 0)
1525    *** If LUN ID is omitted or -1, the lowest free one is assigned ***""", type=int, required=False)
1526    p.set_defaults(func=iscsi_target_node_add_lun)
1527
1528    def iscsi_target_node_set_auth(args):
1529        rpc.iscsi.iscsi_target_node_set_auth(
1530            args.client,
1531            name=args.name,
1532            chap_group=args.chap_group,
1533            disable_chap=args.disable_chap,
1534            require_chap=args.require_chap,
1535            mutual_chap=args.mutual_chap)
1536
1537    p = subparsers.add_parser('iscsi_target_node_set_auth',
1538                              help='Set CHAP authentication for the target node')
1539    p.add_argument('name', help='Target node name (ASCII)')
1540    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this target node.
1541    *** Authentication group must be precreated ***""", type=int)
1542    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this target node.
1543    *** Mutually exclusive with --require-chap ***""", action='store_true')
1544    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this target node.
1545    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1546    p.add_argument('-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.',
1547                   action='store_true')
1548    p.set_defaults(func=iscsi_target_node_set_auth)
1549
1550    def iscsi_target_node_add_pg_ig_maps(args):
1551        pg_ig_maps = []
1552        for u in args.pg_ig_mappings.strip().split(" "):
1553            pg, ig = u.split(":")
1554            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1555        rpc.iscsi.iscsi_target_node_add_pg_ig_maps(
1556            args.client,
1557            pg_ig_maps=pg_ig_maps,
1558            name=args.name)
1559
1560    p = subparsers.add_parser('iscsi_target_node_add_pg_ig_maps',
1561                              help='Add PG-IG maps to the target node')
1562    p.add_argument('name', help='Target node name (ASCII)')
1563    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1564    Whitespace separated, quoted, mapping defined with colon
1565    separated list of "tags" (int > 0)
1566    Example: '1:1 2:2 2:1'
1567    *** The Portal/Initiator Groups must be precreated ***""")
1568    p.set_defaults(func=iscsi_target_node_add_pg_ig_maps)
1569
1570    def iscsi_target_node_remove_pg_ig_maps(args):
1571        pg_ig_maps = []
1572        for u in args.pg_ig_mappings.strip().split(" "):
1573            pg, ig = u.split(":")
1574            pg_ig_maps.append({"pg_tag": int(pg), "ig_tag": int(ig)})
1575        rpc.iscsi.iscsi_target_node_remove_pg_ig_maps(
1576            args.client, pg_ig_maps=pg_ig_maps, name=args.name)
1577
1578    p = subparsers.add_parser('iscsi_target_node_remove_pg_ig_maps',
1579                              help='Delete PG-IG maps from the target node')
1580    p.add_argument('name', help='Target node name (ASCII)')
1581    p.add_argument('pg_ig_mappings', help="""List of (Portal_Group_Tag:Initiator_Group_Tag) mappings
1582    Whitespace separated, quoted, mapping defined with colon
1583    separated list of "tags" (int > 0)
1584    Example: '1:1 2:2 2:1'
1585    *** The Portal/Initiator Groups must be precreated ***""")
1586    p.set_defaults(func=iscsi_target_node_remove_pg_ig_maps)
1587
1588    def iscsi_target_node_set_redirect(args):
1589        rpc.iscsi.iscsi_target_node_set_redirect(
1590            args.client,
1591            name=args.name,
1592            pg_tag=args.pg_tag,
1593            redirect_host=args.redirect_host,
1594            redirect_port=args.redirect_port)
1595
1596    p = subparsers.add_parser('iscsi_target_node_set_redirect',
1597                              help="""Update redirect portal of the public portal group for the target node.
1598    Omit redirect host and port to clear previously set redirect settings.""")
1599    p.add_argument('name', help='Target node name (ASCII)')
1600    p.add_argument('pg_tag', help='Portal group tag (unique, integer > 0)', type=int)
1601    p.add_argument('-a', '--redirect-host', help='Numeric IP address for redirect portal', required=False)
1602    p.add_argument('-p', '--redirect-port', help='Numeric TCP port for redirect portal', required=False)
1603    p.set_defaults(func=iscsi_target_node_set_redirect)
1604
1605    def iscsi_target_node_request_logout(args):
1606        rpc.iscsi.iscsi_target_node_request_logout(
1607            args.client,
1608            name=args.name,
1609            pg_tag=args.pg_tag)
1610
1611    p = subparsers.add_parser('iscsi_target_node_request_logout',
1612                              help="""For the target node, request connections whose portal group tag
1613    match to logout, or request all connections if portal group tag is omitted.""")
1614    p.add_argument('name', help='Target node name (ASCII)')
1615    p.add_argument('-t', '--pg-tag', help='Portal group tag (unique, integer > 0)', type=int, required=False)
1616    p.set_defaults(func=iscsi_target_node_request_logout)
1617
1618    def iscsi_create_portal_group(args):
1619        portals = []
1620        for p in args.portal_list.strip().split(' '):
1621            ip, separator, port_cpumask = p.rpartition(':')
1622            split_port_cpumask = port_cpumask.split('@')
1623            if len(split_port_cpumask) == 1:
1624                port = port_cpumask
1625                portals.append({'host': ip, 'port': port})
1626            else:
1627                port = split_port_cpumask[0]
1628                cpumask = split_port_cpumask[1]
1629                portals.append({'host': ip, 'port': port})
1630                print("WARNING: Specifying a portal group with a CPU mask is no longer supported. Ignoring it.")
1631        rpc.iscsi.iscsi_create_portal_group(
1632            args.client,
1633            portals=portals,
1634            tag=args.tag,
1635            private=args.private,
1636            wait=args.wait)
1637
1638    p = subparsers.add_parser('iscsi_create_portal_group',
1639                              help='Add a portal group')
1640    p.add_argument(
1641        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1642    p.add_argument('portal_list', help="""List of portals in host:port format, separated by whitespace
1643    Example: '192.168.100.100:3260 192.168.100.100:3261 192.168.100.100:3262""")
1644    p.add_argument('-p', '--private', help="""Public (false) or private (true) portal group.
1645    Private portal groups do not have their portals returned by a discovery session. A public
1646    portal group may optionally specify a redirect portal for non-discovery logins. This redirect
1647    portal must be from a private portal group.""", action='store_true')
1648    p.add_argument('-w', '--wait', help="""Do not listening on portals until it is started explicitly.
1649    One major iSCSI initiator may not retry login once it failed. Hence for such initiator, listening
1650    on portals should be allowed after all associated target nodes are created.""", action='store_true')
1651    p.set_defaults(func=iscsi_create_portal_group)
1652
1653    def iscsi_start_portal_group(args):
1654        rpc.iscsi.iscsi_start_portal_group(args.client, tag=args.tag)
1655
1656    p = subparsers.add_parser('iscsi_start_portal_group',
1657                              help='Start listening on portals if it is not started yet.')
1658    p.add_argument(
1659        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1660    p.set_defaults(func=iscsi_start_portal_group)
1661
1662    def iscsi_create_initiator_group(args):
1663        initiators = []
1664        netmasks = []
1665        for i in args.initiator_list.strip().split(' '):
1666            initiators.append(i)
1667        for n in args.netmask_list.strip().split(' '):
1668            netmasks.append(n)
1669        rpc.iscsi.iscsi_create_initiator_group(
1670            args.client,
1671            tag=args.tag,
1672            initiators=initiators,
1673            netmasks=netmasks)
1674
1675    p = subparsers.add_parser('iscsi_create_initiator_group',
1676                              help='Add an initiator group')
1677    p.add_argument(
1678        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1679    p.add_argument('initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1680    enclosed in quotes.  Example: 'ANY' or 'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""")
1681    p.add_argument('netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1682    Example: '255.255.0.0 255.248.0.0' etc""")
1683    p.set_defaults(func=iscsi_create_initiator_group)
1684
1685    def iscsi_initiator_group_add_initiators(args):
1686        initiators = None
1687        netmasks = None
1688        if args.initiator_list:
1689            initiators = []
1690            for i in args.initiator_list.strip().split(' '):
1691                initiators.append(i)
1692        if args.netmask_list:
1693            netmasks = []
1694            for n in args.netmask_list.strip().split(' '):
1695                netmasks.append(n)
1696        rpc.iscsi.iscsi_initiator_group_add_initiators(
1697            args.client,
1698            tag=args.tag,
1699            initiators=initiators,
1700            netmasks=netmasks)
1701
1702    p = subparsers.add_parser('iscsi_initiator_group_add_initiators',
1703                              help='Add initiators to an existing initiator group')
1704    p.add_argument(
1705        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1706    p.add_argument('-n', dest='initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1707    enclosed in quotes.  This parameter can be omitted.  Example: 'ANY' or
1708    'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""", required=False)
1709    p.add_argument('-m', dest='netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1710    This parameter can be omitted.  Example: '255.255.0.0 255.248.0.0' etc""", required=False)
1711    p.set_defaults(func=iscsi_initiator_group_add_initiators)
1712
1713    def iscsi_initiator_group_remove_initiators(args):
1714        initiators = None
1715        netmasks = None
1716        if args.initiator_list:
1717            initiators = []
1718            for i in args.initiator_list.strip().split(' '):
1719                initiators.append(i)
1720        if args.netmask_list:
1721            netmasks = []
1722            for n in args.netmask_list.strip().split(' '):
1723                netmasks.append(n)
1724        rpc.iscsi.iscsi_initiator_group_remove_initiators(
1725            args.client,
1726            tag=args.tag,
1727            initiators=initiators,
1728            netmasks=netmasks)
1729
1730    p = subparsers.add_parser('iscsi_initiator_group_remove_initiators',
1731                              help='Delete initiators from an existing initiator group')
1732    p.add_argument(
1733        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1734    p.add_argument('-n', dest='initiator_list', help="""Whitespace-separated list of initiator hostnames or IP addresses,
1735    enclosed in quotes.  This parameter can be omitted.  Example: 'ANY' or
1736    'iqn.2016-06.io.spdk:host1 iqn.2016-06.io.spdk:host2'""", required=False)
1737    p.add_argument('-m', dest='netmask_list', help="""Whitespace-separated list of initiator netmasks enclosed in quotes.
1738    This parameter can be omitted.  Example: '255.255.0.0 255.248.0.0' etc""", required=False)
1739    p.set_defaults(func=iscsi_initiator_group_remove_initiators)
1740
1741    def iscsi_delete_target_node(args):
1742        rpc.iscsi.iscsi_delete_target_node(
1743            args.client, target_node_name=args.target_node_name)
1744
1745    p = subparsers.add_parser('iscsi_delete_target_node',
1746                              help='Delete a target node')
1747    p.add_argument('target_node_name',
1748                   help='Target node name to be deleted. Example: iqn.2016-06.io.spdk:disk1.')
1749    p.set_defaults(func=iscsi_delete_target_node)
1750
1751    def iscsi_delete_portal_group(args):
1752        rpc.iscsi.iscsi_delete_portal_group(args.client, tag=args.tag)
1753
1754    p = subparsers.add_parser('iscsi_delete_portal_group',
1755                              help='Delete a portal group')
1756    p.add_argument(
1757        'tag', help='Portal group tag (unique, integer > 0)', type=int)
1758    p.set_defaults(func=iscsi_delete_portal_group)
1759
1760    def iscsi_delete_initiator_group(args):
1761        rpc.iscsi.iscsi_delete_initiator_group(args.client, tag=args.tag)
1762
1763    p = subparsers.add_parser('iscsi_delete_initiator_group',
1764                              help='Delete an initiator group')
1765    p.add_argument(
1766        'tag', help='Initiator group tag (unique, integer > 0)', type=int)
1767    p.set_defaults(func=iscsi_delete_initiator_group)
1768
1769    def iscsi_portal_group_set_auth(args):
1770        rpc.iscsi.iscsi_portal_group_set_auth(
1771            args.client,
1772            tag=args.tag,
1773            chap_group=args.chap_group,
1774            disable_chap=args.disable_chap,
1775            require_chap=args.require_chap,
1776            mutual_chap=args.mutual_chap)
1777
1778    p = subparsers.add_parser('iscsi_portal_group_set_auth',
1779                              help='Set CHAP authentication for discovery sessions specific for the portal group')
1780    p.add_argument('tag', help='Portal group tag (unique, integer > 0)', type=int)
1781    p.add_argument('-g', '--chap-group', help="""Authentication group ID for this portal group.
1782    *** Authentication group must be precreated ***""", type=int)
1783    p.add_argument('-d', '--disable-chap', help="""CHAP authentication should be disabled for this portal group.
1784    *** Mutually exclusive with --require-chap ***""", action='store_true')
1785    p.add_argument('-r', '--require-chap', help="""CHAP authentication should be required for this portal group.
1786    *** Mutually exclusive with --disable-chap ***""", action='store_true')
1787    p.add_argument('-m', '--mutual-chap', help='CHAP authentication should be mutual/bidirectional.',
1788                   action='store_true')
1789    p.set_defaults(func=iscsi_portal_group_set_auth)
1790
1791    def iscsi_get_connections(args):
1792        print_dict(rpc.iscsi.iscsi_get_connections(args.client))
1793
1794    p = subparsers.add_parser('iscsi_get_connections',
1795                              help='Display iSCSI connections')
1796    p.set_defaults(func=iscsi_get_connections)
1797
1798    def iscsi_get_options(args):
1799        print_dict(rpc.iscsi.iscsi_get_options(args.client))
1800
1801    p = subparsers.add_parser('iscsi_get_options',
1802                              help='Display iSCSI global parameters')
1803    p.set_defaults(func=iscsi_get_options)
1804
1805    def scsi_get_devices(args):
1806        print_dict(rpc.iscsi.scsi_get_devices(args.client))
1807
1808    p = subparsers.add_parser('scsi_get_devices', help='Display SCSI devices')
1809    p.set_defaults(func=scsi_get_devices)
1810
1811    # trace
1812    def trace_enable_tpoint_group(args):
1813        rpc.trace.trace_enable_tpoint_group(args.client, name=args.name)
1814
1815    p = subparsers.add_parser('trace_enable_tpoint_group',
1816                              help='enable trace on a specific tpoint group')
1817    p.add_argument(
1818        'name', help="""trace group name we want to enable in tpoint_group_mask.
1819        (for example "bdev" for bdev trace group, "all" for all trace groups).""")
1820    p.set_defaults(func=trace_enable_tpoint_group)
1821
1822    def trace_disable_tpoint_group(args):
1823        rpc.trace.trace_disable_tpoint_group(args.client, name=args.name)
1824
1825    p = subparsers.add_parser('trace_disable_tpoint_group',
1826                              help='disable trace on a specific tpoint group')
1827    p.add_argument(
1828        'name', help="""trace group name we want to disable in tpoint_group_mask.
1829        (for example "bdev" for bdev trace group, "all" for all trace groups).""")
1830    p.set_defaults(func=trace_disable_tpoint_group)
1831
1832    def trace_set_tpoint_mask(args):
1833        rpc.trace.trace_set_tpoint_mask(args.client, name=args.name, tpoint_mask=args.tpoint_mask)
1834
1835    p = subparsers.add_parser('trace_set_tpoint_mask',
1836                              help='enable tracepoint mask on a specific tpoint group')
1837    p.add_argument(
1838        'name', help="""trace group name we want to enable in tpoint_group_mask.
1839        (for example "bdev" for bdev trace group)""")
1840    p.add_argument(
1841        'tpoint_mask', help="""tracepoints to be enabled inside a given trace group.
1842        (for example value of "0x3" will enable only the first two tpoints in this group)""",
1843        type=lambda m: int(m, 16))
1844    p.set_defaults(func=trace_set_tpoint_mask)
1845
1846    def trace_clear_tpoint_mask(args):
1847        rpc.trace.trace_clear_tpoint_mask(args.client, name=args.name, tpoint_mask=args.tpoint_mask)
1848
1849    p = subparsers.add_parser('trace_clear_tpoint_mask',
1850                              help='disable tracepoint mask on a specific tpoint group')
1851    p.add_argument(
1852        'name', help="""trace group name we want to disable in tpoint_group_mask.
1853        (for example "bdev" for bdev trace group)""")
1854    p.add_argument(
1855        'tpoint_mask', help="""tracepoints to be disabled inside a given trace group.
1856        (for example value of "0x3" will disable the first two tpoints in this group)""",
1857        type=lambda m: int(m, 16))
1858    p.set_defaults(func=trace_clear_tpoint_mask)
1859
1860    def trace_get_tpoint_group_mask(args):
1861        print_dict(rpc.trace.trace_get_tpoint_group_mask(args.client))
1862
1863    p = subparsers.add_parser('trace_get_tpoint_group_mask', help='get trace point group mask')
1864    p.set_defaults(func=trace_get_tpoint_group_mask)
1865
1866    def trace_get_info(args):
1867        print_dict(rpc.trace.trace_get_info(args.client))
1868
1869    p = subparsers.add_parser('trace_get_info',
1870                              help='get name of shared memory file and list of the available trace point groups')
1871    p.set_defaults(func=trace_get_info)
1872
1873    # log
1874    def log_set_flag(args):
1875        rpc.log.log_set_flag(args.client, flag=args.flag)
1876
1877    p = subparsers.add_parser('log_set_flag', help='set log flag')
1878    p.add_argument(
1879        'flag', help='log flag we want to set. (for example "nvme").')
1880    p.set_defaults(func=log_set_flag)
1881
1882    def log_clear_flag(args):
1883        rpc.log.log_clear_flag(args.client, flag=args.flag)
1884
1885    p = subparsers.add_parser('log_clear_flag', help='clear log flag')
1886    p.add_argument(
1887        'flag', help='log flag we want to clear. (for example "nvme").')
1888    p.set_defaults(func=log_clear_flag)
1889
1890    def log_get_flags(args):
1891        print_dict(rpc.log.log_get_flags(args.client))
1892
1893    p = subparsers.add_parser('log_get_flags', help='get log flags')
1894    p.set_defaults(func=log_get_flags)
1895
1896    def log_set_level(args):
1897        rpc.log.log_set_level(args.client, level=args.level)
1898
1899    p = subparsers.add_parser('log_set_level', help='set log level')
1900    p.add_argument('level', help='log level we want to set. (for example "DEBUG").')
1901    p.set_defaults(func=log_set_level)
1902
1903    def log_get_level(args):
1904        print_dict(rpc.log.log_get_level(args.client))
1905
1906    p = subparsers.add_parser('log_get_level', help='get log level')
1907    p.set_defaults(func=log_get_level)
1908
1909    def log_set_print_level(args):
1910        rpc.log.log_set_print_level(args.client, level=args.level)
1911
1912    p = subparsers.add_parser('log_set_print_level', help='set log print level')
1913    p.add_argument('level', help='log print level we want to set. (for example "DEBUG").')
1914    p.set_defaults(func=log_set_print_level)
1915
1916    def log_get_print_level(args):
1917        print_dict(rpc.log.log_get_print_level(args.client))
1918
1919    p = subparsers.add_parser('log_get_print_level', help='get log print level')
1920    p.set_defaults(func=log_get_print_level)
1921
1922    # lvol
1923    def bdev_lvol_create_lvstore(args):
1924        print_json(rpc.lvol.bdev_lvol_create_lvstore(args.client,
1925                                                     bdev_name=args.bdev_name,
1926                                                     lvs_name=args.lvs_name,
1927                                                     cluster_sz=args.cluster_sz,
1928                                                     clear_method=args.clear_method,
1929                                                     num_md_pages_per_cluster_ratio=args.md_pages_per_cluster_ratio))
1930
1931    p = subparsers.add_parser('bdev_lvol_create_lvstore', help='Add logical volume store on base bdev')
1932    p.add_argument('bdev_name', help='base bdev name')
1933    p.add_argument('lvs_name', help='name for lvol store')
1934    p.add_argument('-c', '--cluster-sz', help='size of cluster (in bytes)', type=int, required=False)
1935    p.add_argument('--clear-method', help="""Change clear method for data region.
1936        Available: none, unmap, write_zeroes""", required=False)
1937    p.add_argument('-m', '--md-pages-per-cluster-ratio', help='reserved metadata pages for each cluster', type=int, required=False)
1938    p.set_defaults(func=bdev_lvol_create_lvstore)
1939
1940    def bdev_lvol_rename_lvstore(args):
1941        rpc.lvol.bdev_lvol_rename_lvstore(args.client,
1942                                          old_name=args.old_name,
1943                                          new_name=args.new_name)
1944
1945    p = subparsers.add_parser('bdev_lvol_rename_lvstore', help='Change logical volume store name')
1946    p.add_argument('old_name', help='old name')
1947    p.add_argument('new_name', help='new name')
1948    p.set_defaults(func=bdev_lvol_rename_lvstore)
1949
1950    def bdev_lvol_grow_lvstore(args):
1951        print_dict(rpc.lvol.bdev_lvol_grow_lvstore(args.client,
1952                                                   uuid=args.uuid,
1953                                                   lvs_name=args.lvs_name))
1954
1955    p = subparsers.add_parser('bdev_lvol_grow_lvstore',
1956                              help='Grow the lvstore size to the underlying bdev size')
1957    p.add_argument('-u', '--uuid', help='lvol store UUID', required=False)
1958    p.add_argument('-l', '--lvs-name', help='lvol store name', required=False)
1959    p.set_defaults(func=bdev_lvol_grow_lvstore)
1960
1961    def bdev_lvol_create(args):
1962        print_json(rpc.lvol.bdev_lvol_create(args.client,
1963                                             lvol_name=args.lvol_name,
1964                                             size_in_mib=args.size_in_mib,
1965                                             thin_provision=args.thin_provision,
1966                                             clear_method=args.clear_method,
1967                                             uuid=args.uuid,
1968                                             lvs_name=args.lvs_name))
1969
1970    p = subparsers.add_parser('bdev_lvol_create', help='Add a bdev with an logical volume backend')
1971    p.add_argument('-u', '--uuid', help='lvol store UUID', required=False)
1972    p.add_argument('-l', '--lvs-name', help='lvol store name', required=False)
1973    p.add_argument('-t', '--thin-provision', action='store_true', help='create lvol bdev as thin provisioned')
1974    p.add_argument('-c', '--clear-method', help="""Change default data clusters clear method.
1975        Available: none, unmap, write_zeroes""", required=False)
1976    p.add_argument('lvol_name', help='name for this lvol')
1977    p.add_argument('size_in_mib', help='size in MiB for this bdev', type=int)
1978    p.set_defaults(func=bdev_lvol_create)
1979
1980    def bdev_lvol_snapshot(args):
1981        print_json(rpc.lvol.bdev_lvol_snapshot(args.client,
1982                                               lvol_name=args.lvol_name,
1983                                               snapshot_name=args.snapshot_name))
1984
1985    p = subparsers.add_parser('bdev_lvol_snapshot', help='Create a snapshot of an lvol bdev')
1986    p.add_argument('lvol_name', help='lvol bdev name')
1987    p.add_argument('snapshot_name', help='lvol snapshot name')
1988    p.set_defaults(func=bdev_lvol_snapshot)
1989
1990    def bdev_lvol_clone(args):
1991        print_json(rpc.lvol.bdev_lvol_clone(args.client,
1992                                            snapshot_name=args.snapshot_name,
1993                                            clone_name=args.clone_name))
1994
1995    p = subparsers.add_parser('bdev_lvol_clone', help='Create a clone of an lvol snapshot')
1996    p.add_argument('snapshot_name', help='lvol snapshot name')
1997    p.add_argument('clone_name', help='lvol clone name')
1998    p.set_defaults(func=bdev_lvol_clone)
1999
2000    def bdev_lvol_clone_bdev(args):
2001        print_json(rpc.lvol.bdev_lvol_clone_bdev(args.client,
2002                                                 bdev=args.bdev,
2003                                                 lvs_name=args.lvs_name,
2004                                                 clone_name=args.clone_name))
2005
2006    p = subparsers.add_parser('bdev_lvol_clone_bdev',
2007                              help='Create a clone of a non-lvol bdev')
2008    p.add_argument('bdev', help='bdev to clone')
2009    p.add_argument('lvs_name', help='logical volume store name')
2010    p.add_argument('clone_name', help='lvol clone name')
2011    p.set_defaults(func=bdev_lvol_clone_bdev)
2012
2013    def bdev_lvol_rename(args):
2014        rpc.lvol.bdev_lvol_rename(args.client,
2015                                  old_name=args.old_name,
2016                                  new_name=args.new_name)
2017
2018    p = subparsers.add_parser('bdev_lvol_rename', help='Change lvol bdev name')
2019    p.add_argument('old_name', help='lvol bdev name')
2020    p.add_argument('new_name', help='new lvol name')
2021    p.set_defaults(func=bdev_lvol_rename)
2022
2023    def bdev_lvol_inflate(args):
2024        rpc.lvol.bdev_lvol_inflate(args.client,
2025                                   name=args.name)
2026
2027    p = subparsers.add_parser('bdev_lvol_inflate', help='Make thin provisioned lvol a thick provisioned lvol')
2028    p.add_argument('name', help='lvol bdev name')
2029    p.set_defaults(func=bdev_lvol_inflate)
2030
2031    def bdev_lvol_decouple_parent(args):
2032        rpc.lvol.bdev_lvol_decouple_parent(args.client,
2033                                           name=args.name)
2034
2035    p = subparsers.add_parser('bdev_lvol_decouple_parent', help='Decouple parent of lvol')
2036    p.add_argument('name', help='lvol bdev name')
2037    p.set_defaults(func=bdev_lvol_decouple_parent)
2038
2039    def bdev_lvol_resize(args):
2040        rpc.lvol.bdev_lvol_resize(args.client,
2041                                  name=args.name,
2042                                  size_in_mib=args.size_in_mib)
2043
2044    p = subparsers.add_parser('bdev_lvol_resize', help='Resize existing lvol bdev')
2045    p.add_argument('name', help='lvol bdev name')
2046    p.add_argument('size_in_mib', help='new size in MiB for this bdev', type=int)
2047    p.set_defaults(func=bdev_lvol_resize)
2048
2049    def bdev_lvol_set_read_only(args):
2050        rpc.lvol.bdev_lvol_set_read_only(args.client,
2051                                         name=args.name)
2052
2053    p = subparsers.add_parser('bdev_lvol_set_read_only', help='Mark lvol bdev as read only')
2054    p.add_argument('name', help='lvol bdev name')
2055    p.set_defaults(func=bdev_lvol_set_read_only)
2056
2057    def bdev_lvol_delete(args):
2058        rpc.lvol.bdev_lvol_delete(args.client,
2059                                  name=args.name)
2060
2061    p = subparsers.add_parser('bdev_lvol_delete', help='Destroy a logical volume')
2062    p.add_argument('name', help='lvol bdev name')
2063    p.set_defaults(func=bdev_lvol_delete)
2064
2065    def bdev_lvol_delete_lvstore(args):
2066        rpc.lvol.bdev_lvol_delete_lvstore(args.client,
2067                                          uuid=args.uuid,
2068                                          lvs_name=args.lvs_name)
2069
2070    p = subparsers.add_parser('bdev_lvol_delete_lvstore', help='Destroy an logical volume store')
2071    p.add_argument('-u', '--uuid', help='lvol store UUID', required=False)
2072    p.add_argument('-l', '--lvs-name', help='lvol store name', required=False)
2073    p.set_defaults(func=bdev_lvol_delete_lvstore)
2074
2075    def bdev_lvol_get_lvstores(args):
2076        print_dict(rpc.lvol.bdev_lvol_get_lvstores(args.client,
2077                                                   uuid=args.uuid,
2078                                                   lvs_name=args.lvs_name))
2079
2080    p = subparsers.add_parser('bdev_lvol_get_lvstores', help='Display current logical volume store list')
2081    p.add_argument('-u', '--uuid', help='lvol store UUID', required=False)
2082    p.add_argument('-l', '--lvs-name', help='lvol store name', required=False)
2083    p.set_defaults(func=bdev_lvol_get_lvstores)
2084
2085    def bdev_lvol_get_lvols(args):
2086        print_dict(rpc.lvol.bdev_lvol_get_lvols(args.client,
2087                                                lvs_uuid=args.lvs_uuid,
2088                                                lvs_name=args.lvs_name))
2089
2090    p = subparsers.add_parser('bdev_lvol_get_lvols', help='Display current logical volume list')
2091    p.add_argument('-u', '--lvs-uuid', help='only lvols in  lvol store UUID', required=False)
2092    p.add_argument('-l', '--lvs-name', help='only lvols in lvol store name', required=False)
2093    p.set_defaults(func=bdev_lvol_get_lvols)
2094
2095    def bdev_raid_get_bdevs(args):
2096        print_json(rpc.bdev.bdev_raid_get_bdevs(args.client,
2097                                                category=args.category))
2098
2099    p = subparsers.add_parser('bdev_raid_get_bdevs',
2100                              help="""This is used to list all the raid bdev details based on the input category
2101    requested. Category should be one of 'all', 'online', 'configuring' or 'offline'. 'all' means all the raid bdevs whether
2102    they are online or configuring or offline. 'online' is the raid bdev which is registered with bdev layer. 'configuring'
2103    is the raid bdev which does not have full configuration discovered yet. 'offline' is the raid bdev which is not registered
2104    with bdev as of now and it has encountered any error or user has requested to offline the raid bdev""")
2105    p.add_argument('category', help='all or online or configuring or offline')
2106    p.set_defaults(func=bdev_raid_get_bdevs)
2107
2108    def bdev_raid_create(args):
2109        base_bdevs = []
2110        for u in args.base_bdevs.strip().split(" "):
2111            base_bdevs.append(u)
2112
2113        rpc.bdev.bdev_raid_create(args.client,
2114                                  name=args.name,
2115                                  strip_size_kb=args.strip_size_kb,
2116                                  raid_level=args.raid_level,
2117                                  base_bdevs=base_bdevs,
2118                                  uuid=args.uuid)
2119    p = subparsers.add_parser('bdev_raid_create', help='Create new raid bdev')
2120    p.add_argument('-n', '--name', help='raid bdev name', required=True)
2121    p.add_argument('-z', '--strip-size-kb', help='strip size in KB', type=int)
2122    p.add_argument('-r', '--raid-level', help='raid level, raid0, raid1 and a special level concat are supported', required=True)
2123    p.add_argument('-b', '--base-bdevs', help='base bdevs name, whitespace separated list in quotes', required=True)
2124    p.add_argument('--uuid', help='UUID for this raid bdev', required=False)
2125    p.set_defaults(func=bdev_raid_create)
2126
2127    def bdev_raid_delete(args):
2128        rpc.bdev.bdev_raid_delete(args.client,
2129                                  name=args.name)
2130    p = subparsers.add_parser('bdev_raid_delete', help='Delete existing raid bdev')
2131    p.add_argument('name', help='raid bdev name')
2132    p.set_defaults(func=bdev_raid_delete)
2133
2134    def bdev_raid_remove_base_bdev(args):
2135        rpc.bdev.bdev_raid_remove_base_bdev(args.client,
2136                                            name=args.name)
2137    p = subparsers.add_parser('bdev_raid_remove_base_bdev', help='Remove base bdev from existing raid bdev')
2138    p.add_argument('name', help='base bdev name')
2139    p.set_defaults(func=bdev_raid_remove_base_bdev)
2140
2141    # split
2142    def bdev_split_create(args):
2143        print_array(rpc.bdev.bdev_split_create(args.client,
2144                                               base_bdev=args.base_bdev,
2145                                               split_count=args.split_count,
2146                                               split_size_mb=args.split_size_mb))
2147
2148    p = subparsers.add_parser('bdev_split_create',
2149                              help="""Add given disk name to split config. If bdev with base_name
2150    name exist the split bdevs will be created right away, if not split bdevs will be created when base bdev became
2151    available (during examination process).""")
2152    p.add_argument('base_bdev', help='base bdev name')
2153    p.add_argument('-s', '--split-size-mb', help='size in MiB for each bdev', type=int)
2154    p.add_argument('split_count', help="""Optional - number of split bdevs to create. Total size * split_count must not
2155    exceed the base bdev size.""", type=int)
2156    p.set_defaults(func=bdev_split_create)
2157
2158    def bdev_split_delete(args):
2159        rpc.bdev.bdev_split_delete(args.client,
2160                                   base_bdev=args.base_bdev)
2161
2162    p = subparsers.add_parser('bdev_split_delete', help="""Delete split config with all created splits.""")
2163    p.add_argument('base_bdev', help='base bdev name')
2164    p.set_defaults(func=bdev_split_delete)
2165
2166    # ftl
2167    def bdev_ftl_create(args):
2168        print_dict(rpc.bdev.bdev_ftl_create(args.client,
2169                                            name=args.name,
2170                                            base_bdev=args.base_bdev,
2171                                            uuid=args.uuid,
2172                                            cache=args.cache,
2173                                            overprovisioning=args.overprovisioning,
2174                                            l2p_dram_limit=args.l2p_dram_limit,
2175                                            core_mask=args.core_mask,
2176                                            fast_shutdown=args.fast_shutdown))
2177
2178    p = subparsers.add_parser('bdev_ftl_create', help='Add FTL bdev')
2179    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2180    p.add_argument('-d', '--base-bdev', help='Name of bdev used as underlying device',
2181                   required=True)
2182    p.add_argument('-u', '--uuid', help='UUID of restored bdev (not applicable when creating new '
2183                   'instance): e.g. b286d19a-0059-4709-abcd-9f7732b1567d (optional)')
2184    p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache',
2185                   required=True)
2186    p.add_argument('--overprovisioning', help='Percentage of device used for relocation, not exposed'
2187                   ' to user (optional); default 20', type=int)
2188    p.add_argument('--l2p-dram-limit', help='l2p size that could reside in DRAM (optional); default 2048',
2189                   type=int)
2190    p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, '
2191                   'by default core thread will be set to the main application core (optional)')
2192    p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true')
2193    p.set_defaults(func=bdev_ftl_create)
2194
2195    def bdev_ftl_load(args):
2196        print_dict(rpc.bdev.bdev_ftl_load(args.client,
2197                                          name=args.name,
2198                                          base_bdev=args.base_bdev,
2199                                          uuid=args.uuid,
2200                                          cache=args.cache,
2201                                          overprovisioning=args.overprovisioning,
2202                                          l2p_dram_limit=args.l2p_dram_limit,
2203                                          core_mask=args.core_mask,
2204                                          fast_shutdown=args.fast_shutdown))
2205
2206    p = subparsers.add_parser('bdev_ftl_load', help='Load FTL bdev')
2207    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2208    p.add_argument('-d', '--base-bdev', help='Name of bdev used as underlying device',
2209                   required=True)
2210    p.add_argument('-u', '--uuid', help='UUID of restored bdev', required=True)
2211    p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache',
2212                   required=True)
2213    p.add_argument('--overprovisioning', help='Percentage of device used for relocation, not exposed'
2214                   ' to user (optional); default 20', type=int)
2215    p.add_argument('--l2p-dram-limit', help='l2p size that could reside in DRAM (optional); default 2048',
2216                   type=int)
2217    p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, '
2218                   'by default core thread will be set to the main application core (optional)')
2219    p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true')
2220    p.set_defaults(func=bdev_ftl_load)
2221
2222    def bdev_ftl_unload(args):
2223        print_dict(rpc.bdev.bdev_ftl_unload(args.client, name=args.name, fast_shutdown=args.fast_shutdown))
2224
2225    p = subparsers.add_parser('bdev_ftl_unload', help='Unload FTL bdev')
2226    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2227    p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true')
2228    p.set_defaults(func=bdev_ftl_unload)
2229
2230    def bdev_ftl_delete(args):
2231        print_dict(rpc.bdev.bdev_ftl_delete(args.client, name=args.name, fast_shutdown=args.fast_shutdown))
2232
2233    p = subparsers.add_parser('bdev_ftl_delete', help='Delete FTL bdev')
2234    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2235    p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true')
2236    p.set_defaults(func=bdev_ftl_delete)
2237
2238    def bdev_ftl_unmap(args):
2239        print_dict(rpc.bdev.bdev_ftl_unmap(args.client, name=args.name,
2240                                           lba=args.lba,
2241                                           num_blocks=args.num_blocks))
2242
2243    p = subparsers.add_parser('bdev_ftl_unmap', help='FTL unmap')
2244    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2245    p.add_argument('--lba', help='start LBA', required=True, type=int)
2246    p.add_argument('--num-blocks', help='num blocks', required=True, type=int)
2247    p.set_defaults(func=bdev_ftl_unmap)
2248
2249    def bdev_ftl_get_stats(args):
2250        print_dict(rpc.bdev.bdev_ftl_get_stats(args.client, name=args.name))
2251
2252    p = subparsers.add_parser('bdev_ftl_get_stats', help='print ftl stats')
2253    p.add_argument('-b', '--name', help="Name of the bdev", required=True)
2254    p.set_defaults(func=bdev_ftl_get_stats)
2255
2256    # vmd
2257    def vmd_enable(args):
2258        print_dict(rpc.vmd.vmd_enable(args.client))
2259
2260    p = subparsers.add_parser('vmd_enable', aliases=['enable_vmd'], help='Enable VMD enumeration')
2261    p.set_defaults(func=vmd_enable)
2262
2263    def vmd_remove_device(args):
2264        print_dict(rpc.vmd.vmd_remove_device(args.client, addr=args.addr))
2265
2266    p = subparsers.add_parser('vmd_remove_device', help='Remove a device behind VMD')
2267    p.add_argument('addr', help='Address of the device to remove', type=str)
2268    p.set_defaults(func=vmd_remove_device)
2269
2270    def vmd_rescan(args):
2271        print_dict(rpc.vmd.vmd_rescan(args.client))
2272
2273    p = subparsers.add_parser('vmd_rescan', help='Force a rescan of the devices behind VMD')
2274    p.set_defaults(func=vmd_rescan)
2275
2276    # ublk
2277    def ublk_create_target(args):
2278        rpc.ublk.ublk_create_target(args.client,
2279                                    cpumask=args.cpumask)
2280    p = subparsers.add_parser('ublk_create_target',
2281                              help='Create spdk ublk target for ublk dev')
2282    p.add_argument('-m', '--cpumask', help='cpu mask for ublk dev')
2283    p.set_defaults(func=ublk_create_target)
2284
2285    def ublk_destroy_target(args):
2286        rpc.ublk.ublk_destroy_target(args.client)
2287    p = subparsers.add_parser('ublk_destroy_target',
2288                              help='Destroy spdk ublk target for ublk dev')
2289    p.set_defaults(func=ublk_destroy_target)
2290
2291    def ublk_start_disk(args):
2292        print(rpc.ublk.ublk_start_disk(args.client,
2293                                       bdev_name=args.bdev_name,
2294                                       ublk_id=args.ublk_id,
2295                                       num_queues=args.num_queues,
2296                                       queue_depth=args.queue_depth))
2297
2298    p = subparsers.add_parser('ublk_start_disk',
2299                              help='Export a bdev as a ublk device')
2300    p.add_argument('bdev_name', help='Blockdev name to be exported. Example: Malloc0.')
2301    p.add_argument('ublk_id', help='ublk device id to be assigned. Example: 1.', type=int)
2302    p.add_argument('-q', '--num-queues', help="the total number of queues. Example: 1", type=int, required=False)
2303    p.add_argument('-d', '--queue-depth', help="queue depth. Example: 128", type=int, required=False)
2304    p.set_defaults(func=ublk_start_disk)
2305
2306    def ublk_stop_disk(args):
2307        rpc.ublk.ublk_stop_disk(args.client,
2308                                ublk_id=args.ublk_id)
2309
2310    p = subparsers.add_parser('ublk_stop_disk',
2311                              help='Stop a ublk device')
2312    p.add_argument('ublk_id', help='ublk device id to be deleted. Example: 1.', type=int)
2313    p.set_defaults(func=ublk_stop_disk)
2314
2315    def ublk_get_disks(args):
2316        print_dict(rpc.ublk.ublk_get_disks(args.client,
2317                                           ublk_id=args.ublk_id))
2318
2319    p = subparsers.add_parser('ublk_get_disks',
2320                              help='Display full or specified ublk device list')
2321    p.add_argument('-n', '--ublk-id', help="ublk device id. Example: 1", type=int, required=False)
2322    p.set_defaults(func=ublk_get_disks)
2323
2324    # nbd
2325    def nbd_start_disk(args):
2326        print(rpc.nbd.nbd_start_disk(args.client,
2327                                     bdev_name=args.bdev_name,
2328                                     nbd_device=args.nbd_device))
2329
2330    p = subparsers.add_parser('nbd_start_disk',
2331                              help='Export a bdev as an nbd disk')
2332    p.add_argument('bdev_name', help='Blockdev name to be exported. Example: Malloc0.')
2333    p.add_argument('nbd_device', help='Nbd device name to be assigned. Example: /dev/nbd0.', nargs='?')
2334    p.set_defaults(func=nbd_start_disk)
2335
2336    def nbd_stop_disk(args):
2337        rpc.nbd.nbd_stop_disk(args.client,
2338                              nbd_device=args.nbd_device)
2339
2340    p = subparsers.add_parser('nbd_stop_disk',
2341                              help='Stop an nbd disk')
2342    p.add_argument('nbd_device', help='Nbd device name to be stopped. Example: /dev/nbd0.')
2343    p.set_defaults(func=nbd_stop_disk)
2344
2345    def nbd_get_disks(args):
2346        print_dict(rpc.nbd.nbd_get_disks(args.client,
2347                                         nbd_device=args.nbd_device))
2348
2349    p = subparsers.add_parser('nbd_get_disks',
2350                              help='Display full or specified nbd device list')
2351    p.add_argument('-n', '--nbd-device', help="Path of the nbd device. Example: /dev/nbd0", required=False)
2352    p.set_defaults(func=nbd_get_disks)
2353
2354    # NVMe-oF
2355    def nvmf_set_max_subsystems(args):
2356        rpc.nvmf.nvmf_set_max_subsystems(args.client,
2357                                         max_subsystems=args.max_subsystems)
2358
2359    p = subparsers.add_parser('nvmf_set_max_subsystems',
2360                              help='Set the maximum number of NVMf target subsystems')
2361    p.add_argument('-x', '--max-subsystems', help='Max number of NVMf subsystems', type=int, required=True)
2362    p.set_defaults(func=nvmf_set_max_subsystems)
2363
2364    def nvmf_set_config(args):
2365        rpc.nvmf.nvmf_set_config(args.client,
2366                                 passthru_identify_ctrlr=args.passthru_identify_ctrlr,
2367                                 poll_groups_mask=args.poll_groups_mask,
2368                                 discovery_filter=args.discovery_filter)
2369
2370    p = subparsers.add_parser('nvmf_set_config', help='Set NVMf target config')
2371    p.add_argument('-i', '--passthru-identify-ctrlr', help="""Passthrough fields like serial number and model number
2372    when the controller has a single namespace that is an NVMe bdev""", action='store_true')
2373    p.add_argument('-m', '--poll-groups-mask', help='Set cpumask for NVMf poll groups (optional)', type=str)
2374    p.add_argument('-d', '--discovery-filter', help="""Set discovery filter (optional), possible values are: `match_any` (default) or
2375         comma separated values: `transport`, `address`, `svcid`""", type=str)
2376    p.set_defaults(func=nvmf_set_config)
2377
2378    def nvmf_create_transport(args):
2379        rpc.nvmf.nvmf_create_transport(**vars(args))
2380
2381    p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport')
2382    p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True)
2383    p.add_argument('-g', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2384    p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int)
2385    p.add_argument('-m', '--max-io-qpairs-per-ctrlr', help='Max number of IO qpairs per controller', type=int)
2386    p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int)
2387    p.add_argument('-i', '--max-io-size', help='Max I/O size (bytes)', type=int)
2388    p.add_argument('-u', '--io-unit-size', help='I/O unit size (bytes)', type=int)
2389    p.add_argument('-a', '--max-aq-depth', help='Max number of admin cmds per AQ', type=int)
2390    p.add_argument('-n', '--num-shared-buffers', help='The number of pooled data buffers available to the transport', type=int)
2391    p.add_argument('-b', '--buf-cache-size', help='The number of shared buffers to reserve for each poll group', type=int)
2392    p.add_argument('-z', '--zcopy', action='store_true', help='''Use zero-copy operations if the
2393    underlying bdev supports them''')
2394    p.add_argument('-d', '--num-cqe', help="""The number of CQ entries. Only used when no_srq=true.
2395    Relevant only for RDMA transport""", type=int)
2396    p.add_argument('-s', '--max-srq-depth', help='Max number of outstanding I/O per SRQ. Relevant only for RDMA transport', type=int)
2397    p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport')
2398    p.add_argument('-o', '--c2h-success', action='store_false', help='Disable C2H success optimization. Relevant only for TCP transport')
2399    p.add_argument('-f', '--dif-insert-or-strip', action='store_true', help='Enable DIF insert/strip. Relevant only for TCP transport')
2400    p.add_argument('-y', '--sock-priority', help='The sock priority of the tcp connection. Relevant only for TCP transport', type=int)
2401    p.add_argument('-l', '--acceptor-backlog', help='Pending connections allowed at one time. Relevant only for RDMA transport', type=int)
2402    p.add_argument('-x', '--abort-timeout-sec', help='Abort execution timeout value, in seconds', type=int)
2403    p.add_argument('-w', '--no-wr-batching', action='store_true', help='Disable work requests batching. Relevant only for RDMA transport')
2404    p.add_argument('-e', '--control-msg-num', help="""The number of control messages per poll group.
2405    Relevant only for TCP transport""", type=int)
2406    p.add_argument('-M', '--disable-mappable-bar0', action='store_true', help="""Disable mmap() of BAR0.
2407    Relevant only for VFIO-USER transport""")
2408    p.add_argument('-I', '--disable-adaptive-irq', action='store_true', help="""Disable adaptive interrupt feature.
2409    Relevant only for VFIO-USER transport""")
2410    p.add_argument('-S', '--disable-shadow-doorbells', action='store_true', help="""Disable shadow doorbell support.
2411    Relevant only for VFIO-USER transport""")
2412    p.add_argument('--acceptor-poll-rate', help='Polling interval of the acceptor for incoming connections (usec)', type=int)
2413    p.set_defaults(func=nvmf_create_transport)
2414
2415    def nvmf_get_transports(args):
2416        print_dict(rpc.nvmf.nvmf_get_transports(args.client, trtype=args.trtype, tgt_name=args.tgt_name))
2417
2418    p = subparsers.add_parser('nvmf_get_transports', help='Display nvmf transports or required transport')
2419    p.add_argument('--trtype', help='Transport type (optional)')
2420    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2421    p.set_defaults(func=nvmf_get_transports)
2422
2423    def nvmf_get_subsystems(args):
2424        print_dict(rpc.nvmf.nvmf_get_subsystems(args.client, nqn=args.nqn, tgt_name=args.tgt_name))
2425
2426    p = subparsers.add_parser('nvmf_get_subsystems', help='Display nvmf subsystems or required subsystem')
2427    p.add_argument('nqn', help='Subsystem NQN (optional)', nargs="?")
2428    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2429    p.set_defaults(func=nvmf_get_subsystems)
2430
2431    def nvmf_create_subsystem(args):
2432        rpc.nvmf.nvmf_create_subsystem(args.client,
2433                                       nqn=args.nqn,
2434                                       tgt_name=args.tgt_name,
2435                                       serial_number=args.serial_number,
2436                                       model_number=args.model_number,
2437                                       allow_any_host=args.allow_any_host,
2438                                       max_namespaces=args.max_namespaces,
2439                                       ana_reporting=args.ana_reporting,
2440                                       min_cntlid=args.min_cntlid,
2441                                       max_cntlid=args.max_cntlid)
2442
2443    p = subparsers.add_parser('nvmf_create_subsystem', help='Create an NVMe-oF subsystem')
2444    p.add_argument('nqn', help='Subsystem NQN (ASCII)')
2445    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2446    p.add_argument("-s", "--serial-number", help="""
2447    Format:  'sn' etc
2448    Example: 'SPDK00000000000001'""")
2449    p.add_argument("-d", "--model-number", help="""
2450    Format:  'mn' etc
2451    Example: 'SPDK Controller'""")
2452    p.add_argument("-a", "--allow-any-host", action='store_true', help="Allow any host to connect (don't enforce allowed host NQN list)")
2453    p.add_argument("-m", "--max-namespaces", help="Maximum number of namespaces allowed",
2454                   type=int)
2455    p.add_argument("-r", "--ana-reporting", action='store_true', help="Enable ANA reporting feature")
2456    p.add_argument("-i", "--min_cntlid", help="Minimum controller ID", type=int)
2457    p.add_argument("-I", "--max_cntlid", help="Maximum controller ID", type=int)
2458    p.set_defaults(func=nvmf_create_subsystem)
2459
2460    def nvmf_delete_subsystem(args):
2461        rpc.nvmf.nvmf_delete_subsystem(args.client,
2462                                       nqn=args.subsystem_nqn,
2463                                       tgt_name=args.tgt_name)
2464
2465    p = subparsers.add_parser('nvmf_delete_subsystem', help='Delete a nvmf subsystem')
2466    p.add_argument('subsystem_nqn',
2467                   help='subsystem nqn to be deleted. Example: nqn.2016-06.io.spdk:cnode1.')
2468    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2469    p.set_defaults(func=nvmf_delete_subsystem)
2470
2471    def nvmf_subsystem_add_listener(args):
2472        rpc.nvmf.nvmf_subsystem_add_listener(**vars(args))
2473
2474    p = subparsers.add_parser('nvmf_subsystem_add_listener', help='Add a listener to an NVMe-oF subsystem')
2475    p.add_argument('nqn', help='NVMe-oF subsystem NQN (\'discovery\' can be used as shortcut for discovery NQN)')
2476    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2477    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2478    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2479    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2480    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for RDMA or TCP)')
2481    p.add_argument('-k', '--secure-channel', help='Immediately establish a secure channel', action="store_true")
2482    p.set_defaults(func=nvmf_subsystem_add_listener)
2483
2484    def nvmf_subsystem_remove_listener(args):
2485        rpc.nvmf.nvmf_subsystem_remove_listener(args.client,
2486                                                nqn=args.nqn,
2487                                                trtype=args.trtype,
2488                                                traddr=args.traddr,
2489                                                tgt_name=args.tgt_name,
2490                                                adrfam=args.adrfam,
2491                                                trsvcid=args.trsvcid)
2492
2493    p = subparsers.add_parser('nvmf_subsystem_remove_listener', help='Remove a listener from an NVMe-oF subsystem')
2494    p.add_argument('nqn', help='NVMe-oF subsystem NQN (\'discovery\' can be used as shortcut for discovery NQN)')
2495    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2496    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2497    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2498    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2499    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number (required for TCP and RDMA transport types)')
2500    p.set_defaults(func=nvmf_subsystem_remove_listener)
2501
2502    def nvmf_subsystem_listener_set_ana_state(args):
2503        rpc.nvmf.nvmf_subsystem_listener_set_ana_state(args.client,
2504                                                       nqn=args.nqn,
2505                                                       ana_state=args.ana_state,
2506                                                       trtype=args.trtype,
2507                                                       traddr=args.traddr,
2508                                                       tgt_name=args.tgt_name,
2509                                                       adrfam=args.adrfam,
2510                                                       trsvcid=args.trsvcid,
2511                                                       anagrpid=args.anagrpid)
2512
2513    p = subparsers.add_parser('nvmf_subsystem_listener_set_ana_state', help='Set ANA state of a listener for an NVMe-oF subsystem')
2514    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2515    p.add_argument('-n', '--ana-state', help='ANA state to set: optimized, non_optimized, or inaccessible', required=True)
2516    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
2517    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
2518    p.add_argument('-p', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2519    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
2520    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number')
2521    p.add_argument('-g', '--anagrpid', help='ANA group ID (optional)', type=int)
2522    p.set_defaults(func=nvmf_subsystem_listener_set_ana_state)
2523
2524    def nvmf_subsystem_add_ns(args):
2525        rpc.nvmf.nvmf_subsystem_add_ns(args.client,
2526                                       nqn=args.nqn,
2527                                       bdev_name=args.bdev_name,
2528                                       tgt_name=args.tgt_name,
2529                                       ptpl_file=args.ptpl_file,
2530                                       nsid=args.nsid,
2531                                       nguid=args.nguid,
2532                                       eui64=args.eui64,
2533                                       uuid=args.uuid,
2534                                       anagrpid=args.anagrpid)
2535
2536    p = subparsers.add_parser('nvmf_subsystem_add_ns', help='Add a namespace to an NVMe-oF subsystem')
2537    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2538    p.add_argument('bdev_name', help='The name of the bdev that will back this namespace')
2539    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2540    p.add_argument('-p', '--ptpl-file', help='The persistent reservation storage location (optional)', type=str)
2541    p.add_argument('-n', '--nsid', help='The requested NSID (optional)', type=int)
2542    p.add_argument('-g', '--nguid', help='Namespace globally unique identifier (optional)')
2543    p.add_argument('-e', '--eui64', help='Namespace EUI-64 identifier (optional)')
2544    p.add_argument('-u', '--uuid', help='Namespace UUID (optional)')
2545    p.add_argument('-a', '--anagrpid', help='ANA group ID (optional)', type=int)
2546    p.set_defaults(func=nvmf_subsystem_add_ns)
2547
2548    def nvmf_subsystem_remove_ns(args):
2549        rpc.nvmf.nvmf_subsystem_remove_ns(args.client,
2550                                          nqn=args.nqn,
2551                                          nsid=args.nsid,
2552                                          tgt_name=args.tgt_name)
2553
2554    p = subparsers.add_parser('nvmf_subsystem_remove_ns', help='Remove a namespace to an NVMe-oF subsystem')
2555    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2556    p.add_argument('nsid', help='The requested NSID', type=int)
2557    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2558    p.set_defaults(func=nvmf_subsystem_remove_ns)
2559
2560    def nvmf_subsystem_add_host(args):
2561        rpc.nvmf.nvmf_subsystem_add_host(args.client,
2562                                         nqn=args.nqn,
2563                                         host=args.host,
2564                                         tgt_name=args.tgt_name,
2565                                         psk=args.psk)
2566
2567    p = subparsers.add_parser('nvmf_subsystem_add_host', help='Add a host to an NVMe-oF subsystem')
2568    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2569    p.add_argument('host', help='Host NQN to allow')
2570    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2571    p.add_argument('--psk', help='Path to PSK file for TLS authentication (optional). Only applicable for TCP transport.', type=str)
2572    p.set_defaults(func=nvmf_subsystem_add_host)
2573
2574    def nvmf_subsystem_remove_host(args):
2575        rpc.nvmf.nvmf_subsystem_remove_host(args.client,
2576                                            nqn=args.nqn,
2577                                            host=args.host,
2578                                            tgt_name=args.tgt_name)
2579
2580    p = subparsers.add_parser('nvmf_subsystem_remove_host', help='Remove a host from an NVMe-oF subsystem')
2581    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2582    p.add_argument('host', help='Host NQN to remove')
2583    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2584    p.set_defaults(func=nvmf_subsystem_remove_host)
2585
2586    def nvmf_subsystem_allow_any_host(args):
2587        rpc.nvmf.nvmf_subsystem_allow_any_host(args.client,
2588                                               nqn=args.nqn,
2589                                               disable=args.disable,
2590                                               tgt_name=args.tgt_name)
2591
2592    p = subparsers.add_parser('nvmf_subsystem_allow_any_host', help='Allow any host to connect to the subsystem')
2593    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2594    p.add_argument('-e', '--enable', action='store_true', help='Enable allowing any host')
2595    p.add_argument('-d', '--disable', action='store_true', help='Disable allowing any host')
2596    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2597    p.set_defaults(func=nvmf_subsystem_allow_any_host)
2598
2599    def nvmf_subsystem_get_controllers(args):
2600        print_dict(rpc.nvmf.nvmf_subsystem_get_controllers(args.client,
2601                                                           nqn=args.nqn,
2602                                                           tgt_name=args.tgt_name))
2603
2604    p = subparsers.add_parser('nvmf_subsystem_get_controllers',
2605                              help='Display controllers of an NVMe-oF subsystem.')
2606    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2607    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2608    p.set_defaults(func=nvmf_subsystem_get_controllers)
2609
2610    def nvmf_subsystem_get_qpairs(args):
2611        print_dict(rpc.nvmf.nvmf_subsystem_get_qpairs(args.client,
2612                                                      nqn=args.nqn,
2613                                                      tgt_name=args.tgt_name))
2614
2615    p = subparsers.add_parser('nvmf_subsystem_get_qpairs',
2616                              help='Display queue pairs of an NVMe-oF subsystem.')
2617    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2618    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2619    p.set_defaults(func=nvmf_subsystem_get_qpairs)
2620
2621    def nvmf_subsystem_get_listeners(args):
2622        print_dict(rpc.nvmf.nvmf_subsystem_get_listeners(args.client,
2623                                                         nqn=args.nqn,
2624                                                         tgt_name=args.tgt_name))
2625
2626    p = subparsers.add_parser('nvmf_subsystem_get_listeners',
2627                              help='Display listeners of an NVMe-oF subsystem.')
2628    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
2629    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2630    p.set_defaults(func=nvmf_subsystem_get_listeners)
2631
2632    def nvmf_get_stats(args):
2633        print_dict(rpc.nvmf.nvmf_get_stats(args.client, tgt_name=args.tgt_name))
2634
2635    p = subparsers.add_parser(
2636        'nvmf_get_stats', help='Display current statistics for NVMf subsystem')
2637    p.add_argument('-t', '--tgt-name', help='The name of the parent NVMe-oF target (optional)', type=str)
2638    p.set_defaults(func=nvmf_get_stats)
2639
2640    def nvmf_set_crdt(args):
2641        print_dict(rpc.nvmf.nvmf_set_crdt(args.client, args.crdt1, args.crdt2, args.crdt3))
2642
2643    p = subparsers.add_parser(
2644        'nvmf_set_crdt',
2645        help="""Set the 3 crdt (Command Retry Delay Time) values for NVMf subsystem. All
2646        values are in units of 100 milliseconds (same as the NVM Express specification).""")
2647    p.add_argument('-t1', '--crdt1', help='Command Retry Delay Time 1, in units of 100 milliseconds', type=int)
2648    p.add_argument('-t2', '--crdt2', help='Command Retry Delay Time 2, in units of 100 milliseconds', type=int)
2649    p.add_argument('-t3', '--crdt3', help='Command Retry Delay Time 3, in units of 100 milliseconds', type=int)
2650    p.set_defaults(func=nvmf_set_crdt)
2651
2652    # subsystem
2653    def framework_get_subsystems(args):
2654        print_dict(rpc.subsystem.framework_get_subsystems(args.client))
2655
2656    p = subparsers.add_parser('framework_get_subsystems',
2657                              help="""Print subsystems array in initialization order. Each subsystem
2658    entry contain (unsorted) array of subsystems it depends on.""")
2659    p.set_defaults(func=framework_get_subsystems)
2660
2661    def framework_get_config(args):
2662        print_dict(rpc.subsystem.framework_get_config(args.client, args.name))
2663
2664    p = subparsers.add_parser('framework_get_config', help="""Print subsystem configuration""")
2665    p.add_argument('name', help='Name of subsystem to query')
2666    p.set_defaults(func=framework_get_config)
2667
2668    # vhost
2669    def vhost_controller_set_coalescing(args):
2670        rpc.vhost.vhost_controller_set_coalescing(args.client,
2671                                                  ctrlr=args.ctrlr,
2672                                                  delay_base_us=args.delay_base_us,
2673                                                  iops_threshold=args.iops_threshold)
2674
2675    p = subparsers.add_parser('vhost_controller_set_coalescing', help='Set vhost controller coalescing')
2676    p.add_argument('ctrlr', help='controller name')
2677    p.add_argument('delay_base_us', help='Base delay time', type=int)
2678    p.add_argument('iops_threshold', help='IOPS threshold when coalescing is enabled', type=int)
2679    p.set_defaults(func=vhost_controller_set_coalescing)
2680
2681    def virtio_blk_create_transport(args):
2682        rpc.vhost.virtio_blk_create_transport(**vars(args))
2683
2684    p = subparsers.add_parser('virtio_blk_create_transport',
2685                              help='Create virtio blk transport')
2686    p.add_argument('name', help='transport name')
2687    p.set_defaults(func=virtio_blk_create_transport)
2688
2689    def virtio_blk_get_transports(args):
2690        print_dict(rpc.vhost.virtio_blk_get_transports(args.client, name=args.name))
2691
2692    p = subparsers.add_parser('virtio_blk_get_transports', help='Display virtio-blk transports or requested transport')
2693    p.add_argument('--name', help='Transport name (optional)', type=str)
2694    p.set_defaults(func=virtio_blk_get_transports)
2695
2696    def vhost_create_scsi_controller(args):
2697        rpc.vhost.vhost_create_scsi_controller(args.client,
2698                                               ctrlr=args.ctrlr,
2699                                               cpumask=args.cpumask)
2700
2701    p = subparsers.add_parser('vhost_create_scsi_controller', help='Add new vhost controller')
2702    p.add_argument('ctrlr', help='controller name')
2703    p.add_argument('--cpumask', help='cpu mask for this controller')
2704    p.set_defaults(func=vhost_create_scsi_controller)
2705
2706    def vhost_scsi_controller_add_target(args):
2707        print_json(rpc.vhost.vhost_scsi_controller_add_target(args.client,
2708                                                              ctrlr=args.ctrlr,
2709                                                              scsi_target_num=args.scsi_target_num,
2710                                                              bdev_name=args.bdev_name))
2711
2712    p = subparsers.add_parser('vhost_scsi_controller_add_target', help='Add lun to vhost controller')
2713    p.add_argument('ctrlr', help='controller name where add lun')
2714    p.add_argument('scsi_target_num', help='scsi_target_num', type=int)
2715    p.add_argument('bdev_name', help='bdev name')
2716    p.set_defaults(func=vhost_scsi_controller_add_target)
2717
2718    def vhost_scsi_controller_remove_target(args):
2719        rpc.vhost.vhost_scsi_controller_remove_target(args.client,
2720                                                      ctrlr=args.ctrlr,
2721                                                      scsi_target_num=args.scsi_target_num)
2722
2723    p = subparsers.add_parser('vhost_scsi_controller_remove_target',
2724                              help='Remove target from vhost controller')
2725    p.add_argument('ctrlr', help='controller name to remove target from')
2726    p.add_argument('scsi_target_num', help='scsi_target_num', type=int)
2727    p.set_defaults(func=vhost_scsi_controller_remove_target)
2728
2729    def vhost_create_blk_controller(args):
2730        rpc.vhost.vhost_create_blk_controller(**vars(args))
2731
2732    p = subparsers.add_parser('vhost_create_blk_controller', help='Add a new vhost block controller')
2733    p.add_argument('ctrlr', help='controller name')
2734    p.add_argument('dev_name', help='device name')
2735    p.add_argument('--cpumask', help='cpu mask for this controller')
2736    p.add_argument('--transport', help='virtio blk transport name (default: vhost_user_blk)')
2737    p.add_argument("-r", "--readonly", action='store_true', help='Set controller as read-only')
2738    p.add_argument("-p", "--packed_ring", action='store_true', help='Set controller as packed ring supported')
2739    p.add_argument("-l", "--packed_ring_recovery", action='store_true', help='Enable packed ring live recovery')
2740    p.set_defaults(func=vhost_create_blk_controller)
2741
2742    def vhost_get_controllers(args):
2743        print_dict(rpc.vhost.vhost_get_controllers(args.client, args.name))
2744
2745    p = subparsers.add_parser('vhost_get_controllers', help='List all or specific vhost controller(s)')
2746    p.add_argument('-n', '--name', help="Name of vhost controller", required=False)
2747    p.set_defaults(func=vhost_get_controllers)
2748
2749    def vhost_delete_controller(args):
2750        rpc.vhost.vhost_delete_controller(args.client,
2751                                          ctrlr=args.ctrlr)
2752
2753    p = subparsers.add_parser('vhost_delete_controller', help='Delete a vhost controller')
2754    p.add_argument('ctrlr', help='controller name')
2755    p.set_defaults(func=vhost_delete_controller)
2756
2757    def bdev_virtio_attach_controller(args):
2758        print_array(rpc.vhost.bdev_virtio_attach_controller(args.client,
2759                                                            name=args.name,
2760                                                            trtype=args.trtype,
2761                                                            traddr=args.traddr,
2762                                                            dev_type=args.dev_type,
2763                                                            vq_count=args.vq_count,
2764                                                            vq_size=args.vq_size))
2765
2766    p = subparsers.add_parser('bdev_virtio_attach_controller',
2767                              help="""Attach virtio controller using provided
2768    transport type and device type. This will also create bdevs for any block devices connected to the
2769    controller (for example, SCSI devices for a virtio-scsi controller).
2770    Result is array of added bdevs.""")
2771    p.add_argument('name', help="Use this name as base for new created bdevs")
2772    p.add_argument('-t', '--trtype',
2773                   help='Virtio target transport type: pci or user', required=True)
2774    p.add_argument('-a', '--traddr',
2775                   help='Transport type specific target address: e.g. UNIX domain socket path or BDF', required=True)
2776    p.add_argument('-d', '--dev-type',
2777                   help='Device type: blk or scsi', required=True)
2778    p.add_argument('--vq-count', help='Number of virtual queues to be used.', type=int)
2779    p.add_argument('--vq-size', help='Size of each queue', type=int)
2780    p.set_defaults(func=bdev_virtio_attach_controller)
2781
2782    def bdev_virtio_scsi_get_devices(args):
2783        print_dict(rpc.vhost.bdev_virtio_scsi_get_devices(args.client))
2784
2785    p = subparsers.add_parser('bdev_virtio_scsi_get_devices', help='List all Virtio-SCSI devices.')
2786    p.set_defaults(func=bdev_virtio_scsi_get_devices)
2787
2788    def bdev_virtio_detach_controller(args):
2789        rpc.vhost.bdev_virtio_detach_controller(args.client,
2790                                                name=args.name)
2791
2792    p = subparsers.add_parser('bdev_virtio_detach_controller', help="""Remove a Virtio device
2793    This will delete all bdevs exposed by this device""")
2794    p.add_argument('name', help='Virtio device name. E.g. VirtioUser0')
2795    p.set_defaults(func=bdev_virtio_detach_controller)
2796
2797    def bdev_virtio_blk_set_hotplug(args):
2798        rpc.vhost.bdev_virtio_blk_set_hotplug(args.client, enable=args.enable, period_us=args.period_us)
2799
2800    p = subparsers.add_parser('bdev_virtio_blk_set_hotplug', help='Set hotplug options for bdev virtio_blk type.')
2801    p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)")
2802    p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug")
2803    p.add_argument('-r', '--period-us',
2804                   help='How often the hotplug is processed for insert and remove events', type=int)
2805    p.set_defaults(func=bdev_virtio_blk_set_hotplug)
2806
2807    # vfio-user target
2808    def vfu_tgt_set_base_path(args):
2809        rpc.vfio_user.vfu_tgt_set_base_path(args.client, path=args.path)
2810
2811    p = subparsers.add_parser('vfu_tgt_set_base_path', help='Set socket base path.')
2812    p.add_argument('path', help='socket base path')
2813    p.set_defaults(func=vfu_tgt_set_base_path)
2814
2815    def vfu_virtio_delete_endpoint(args):
2816        rpc.vfio_user.vfu_virtio_delete_endpoint(args.client, name=args.name)
2817
2818    p = subparsers.add_parser('vfu_virtio_delete_endpoint', help='Delete the PCI device via endpoint name.')
2819    p.add_argument('name', help='Endpoint name')
2820    p.set_defaults(func=vfu_virtio_delete_endpoint)
2821
2822    def vfu_virtio_create_blk_endpoint(args):
2823        rpc.vfio_user.vfu_virtio_create_blk_endpoint(args.client,
2824                                                     name=args.name,
2825                                                     bdev_name=args.bdev_name,
2826                                                     cpumask=args.cpumask,
2827                                                     num_queues=args.num_queues,
2828                                                     qsize=args.qsize,
2829                                                     packed_ring=args.packed_ring)
2830
2831    p = subparsers.add_parser('vfu_virtio_create_blk_endpoint', help='Create virtio-blk endpoint.')
2832    p.add_argument('name', help='Name of the endpoint')
2833    p.add_argument('--bdev-name', help='block device name', type=str, required=True)
2834    p.add_argument('--cpumask', help='CPU masks')
2835    p.add_argument('--num-queues', help='number of vrings', type=int, default=0)
2836    p.add_argument('--qsize', help='number of element for each vring', type=int, default=0)
2837    p.add_argument("--packed-ring", action='store_true', help='Enable packed ring')
2838    p.set_defaults(func=vfu_virtio_create_blk_endpoint)
2839
2840    def vfu_virtio_scsi_add_target(args):
2841        rpc.vfio_user.vfu_virtio_scsi_add_target(args.client,
2842                                                 name=args.name,
2843                                                 scsi_target_num=args.scsi_target_num,
2844                                                 bdev_name=args.bdev_name)
2845
2846    p = subparsers.add_parser('vfu_virtio_scsi_add_target', help='Attach a block device to SCSI target of PCI endpoint.')
2847    p.add_argument('name', help='Name of the endpoint')
2848    p.add_argument('--scsi-target-num', help='number of SCSI Target', type=int, required=True)
2849    p.add_argument('--bdev-name', help='block device name', type=str, required=True)
2850    p.set_defaults(func=vfu_virtio_scsi_add_target)
2851
2852    def vfu_virtio_scsi_remove_target(args):
2853        rpc.vfio_user.vfu_virtio_scsi_remove_target(args.client,
2854                                                    name=args.name,
2855                                                    scsi_target_num=args.scsi_target_num)
2856
2857    p = subparsers.add_parser('vfu_virtio_scsi_remove_target', help='Remove the specified SCSI target of PCI endpoint.')
2858    p.add_argument('name', help='Name of the endpoint')
2859    p.add_argument('--scsi-target-num', help='number of SCSI Target', type=int, required=True)
2860    p.set_defaults(func=vfu_virtio_scsi_remove_target)
2861
2862    def vfu_virtio_create_scsi_endpoint(args):
2863        rpc.vfio_user.vfu_virtio_create_scsi_endpoint(args.client,
2864                                                      name=args.name,
2865                                                      cpumask=args.cpumask,
2866                                                      num_io_queues=args.num_io_queues,
2867                                                      qsize=args.qsize,
2868                                                      packed_ring=args.packed_ring)
2869
2870    p = subparsers.add_parser('vfu_virtio_create_scsi_endpoint', help='Create virtio-scsi endpoint.')
2871    p.add_argument('name', help='Name of the endpoint')
2872    p.add_argument('--cpumask', help='CPU masks')
2873    p.add_argument('--num-io-queues', help='number of IO vrings', type=int, default=0)
2874    p.add_argument('--qsize', help='number of element for each vring', type=int, default=0)
2875    p.add_argument("--packed-ring", action='store_true', help='Enable packed ring')
2876    p.set_defaults(func=vfu_virtio_create_scsi_endpoint)
2877
2878    # accel_fw
2879    def accel_get_opc_assignments(args):
2880        print_dict(rpc.accel.accel_get_opc_assignments(args.client))
2881
2882    p = subparsers.add_parser('accel_get_opc_assignments', help='Get list of opcode name to module assignments.')
2883    p.set_defaults(func=accel_get_opc_assignments)
2884
2885    def accel_get_module_info(args):
2886        print_dict(rpc.accel.accel_get_module_info(args.client))
2887
2888    p = subparsers.add_parser('accel_get_module_info', aliases=['accel_get_engine_info'],
2889                              help='Get list of valid module names and their operations.')
2890    p.set_defaults(func=accel_get_module_info)
2891
2892    def accel_assign_opc(args):
2893        rpc.accel.accel_assign_opc(args.client, opname=args.opname, module=args.module)
2894
2895    p = subparsers.add_parser('accel_assign_opc', help='Manually assign an operation to a module.')
2896    p.add_argument('-o', '--opname', help='opname')
2897    p.add_argument('-m', '--module', help='name of module')
2898    p.set_defaults(func=accel_assign_opc)
2899
2900    def accel_crypto_key_create(args):
2901        print_dict(rpc.accel.accel_crypto_key_create(args.client,
2902                                                     cipher=args.cipher,
2903                                                     key=args.key,
2904                                                     key2=args.key2,
2905                                                     tweak_mode=args.tweak_mode,
2906                                                     name=args.name))
2907
2908    p = subparsers.add_parser('accel_crypto_key_create', help='Create encryption key')
2909    p.add_argument('-c', '--cipher', help='cipher', required=True)
2910    p.add_argument('-k', '--key', help='key', required=True)
2911    p.add_argument('-e', '--key2', help='key2', required=False, default=None)
2912    p.add_argument('-t', '--tweak-mode', help='tweak mode', required=False, default=None)
2913    p.add_argument('-n', '--name', help='key name', required=True)
2914    p.set_defaults(func=accel_crypto_key_create)
2915
2916    def accel_crypto_key_destroy(args):
2917        print_dict(rpc.accel.accel_crypto_key_destroy(args.client,
2918                                                      key_name=args.name))
2919
2920    p = subparsers.add_parser('accel_crypto_key_destroy', help='Destroy encryption key')
2921    p.add_argument('-n', '--name', help='key name', required=True, type=str)
2922    p.set_defaults(func=accel_crypto_key_destroy)
2923
2924    def accel_crypto_keys_get(args):
2925        print_dict(rpc.accel.accel_crypto_keys_get(args.client,
2926                                                   key_name=args.key_name))
2927
2928    p = subparsers.add_parser('accel_crypto_keys_get', help='Get a list of the crypto keys')
2929    p.add_argument('-k', '--key-name', help='Get information about a specific key', type=str)
2930    p.set_defaults(func=accel_crypto_keys_get)
2931
2932    def accel_set_driver(args):
2933        rpc.accel.accel_set_driver(args.client, name=args.name)
2934
2935    p = subparsers.add_parser('accel_set_driver', help='Select accel platform driver to execute ' +
2936                              'operation chains')
2937    p.add_argument('name', help='name of the platform driver')
2938    p.set_defaults(func=accel_set_driver)
2939
2940    def accel_set_options(args):
2941        rpc.accel.accel_set_options(args.client, args.small_cache_size, args.large_cache_size,
2942                                    args.task_count, args.sequence_count, args.buf_count)
2943
2944    p = subparsers.add_parser('accel_set_options', help='Set accel framework\'s options')
2945    p.add_argument('--small-cache-size', type=int, help='Size of the small iobuf cache')
2946    p.add_argument('--large-cache-size', type=int, help='Size of the large iobuf cache')
2947    p.add_argument('--task-count', type=int, help='Maximum number of tasks per IO channel')
2948    p.add_argument('--sequence-count', type=int, help='Maximum number of sequences per IO channel')
2949    p.add_argument('--buf-count', type=int, help='Maximum number of buffers per IO channel')
2950    p.set_defaults(func=accel_set_options)
2951
2952    def accel_get_stats(args):
2953        print_dict(rpc.accel.accel_get_stats(args.client))
2954
2955    p = subparsers.add_parser('accel_get_stats', help='Display accel framework\'s statistics')
2956    p.set_defaults(func=accel_get_stats)
2957
2958    # ioat
2959    def ioat_scan_accel_module(args):
2960        rpc.ioat.ioat_scan_accel_module(args.client)
2961
2962    p = subparsers.add_parser('ioat_scan_accel_module', aliases=['ioat_scan_accel_engine'],
2963                              help='Enable IOAT accel module offload.')
2964    p.set_defaults(func=ioat_scan_accel_module)
2965
2966    # dpdk compressdev
2967    def compressdev_scan_accel_module(args):
2968        rpc.compressdev.compressdev_scan_accel_module(args.client, pmd=args.pmd)
2969
2970    p = subparsers.add_parser('compressdev_scan_accel_module', help='Scan and enable compressdev module and set pmd option.')
2971    p.add_argument('-p', '--pmd', type=int, help='0 = auto-select, 1= QAT only, 2 = mlx5_pci only')
2972    p.set_defaults(func=compressdev_scan_accel_module)
2973
2974    # dsa
2975    def dsa_scan_accel_module(args):
2976        rpc.dsa.dsa_scan_accel_module(args.client, config_kernel_mode=args.config_kernel_mode)
2977
2978    p = subparsers.add_parser('dsa_scan_accel_module', aliases=['dsa_scan_accel_engine'],
2979                              help='Set config and enable dsa accel module offload.')
2980    p.add_argument('-k', '--config-kernel-mode', help='Use Kernel mode dsa',
2981                   action='store_true', dest='config_kernel_mode')
2982    p.set_defaults(func=dsa_scan_accel_module, config_kernel_mode=None)
2983
2984    # iaa
2985    def iaa_scan_accel_module(args):
2986        rpc.iaa.iaa_scan_accel_module(args.client)
2987
2988    p = subparsers.add_parser('iaa_scan_accel_module', aliases=['iaa_scan_accel_engine'],
2989                              help='Set config and enable iaa accel module offload.')
2990    p.set_defaults(func=iaa_scan_accel_module)
2991
2992    def dpdk_cryptodev_scan_accel_module(args):
2993        rpc.dpdk_cryptodev.dpdk_cryptodev_scan_accel_module(args.client)
2994
2995    p = subparsers.add_parser('dpdk_cryptodev_scan_accel_module',
2996                              help='Enable dpdk_cryptodev accel module offload.')
2997    p.set_defaults(func=dpdk_cryptodev_scan_accel_module)
2998
2999    def dpdk_cryptodev_set_driver(args):
3000        rpc.dpdk_cryptodev.dpdk_cryptodev_set_driver(args.client,
3001                                                     driver_name=args.driver_name)
3002
3003    p = subparsers.add_parser('dpdk_cryptodev_set_driver',
3004                              help='Set the DPDK cryptodev driver.')
3005    p.add_argument('-d', '--driver-name', help='The driver, can be one of crypto_aesni_mb, crypto_qat or mlx5_pci', type=str)
3006    p.set_defaults(func=dpdk_cryptodev_set_driver)
3007
3008    def dpdk_cryptodev_get_driver(args):
3009        print_dict(rpc.dpdk_cryptodev.dpdk_cryptodev_get_driver(args.client))
3010
3011    p = subparsers.add_parser('dpdk_cryptodev_get_driver', help='Get the DPDK cryptodev driver')
3012    p.set_defaults(func=dpdk_cryptodev_get_driver)
3013
3014    # mlx5
3015    def mlx5_scan_accel_module(args):
3016        rpc.mlx5.mlx5_scan_accel_module(args.client,
3017                                        qp_size=args.qp_size,
3018                                        num_requests=args.num_requests)
3019
3020    p = subparsers.add_parser('mlx5_scan_accel_module', help='Enable mlx5 accel module.')
3021    p.add_argument('-q', '--qp-size', type=int, help='QP size')
3022    p.add_argument('-r', '--num-requests', type=int, help='Size of the shared requests pool')
3023    p.set_defaults(func=mlx5_scan_accel_module)
3024
3025    # opal
3026    def bdev_nvme_opal_init(args):
3027        rpc.nvme.bdev_nvme_opal_init(args.client,
3028                                     nvme_ctrlr_name=args.nvme_ctrlr_name,
3029                                     password=args.password)
3030
3031    p = subparsers.add_parser('bdev_nvme_opal_init', help='take ownership and activate')
3032    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name')
3033    p.add_argument('-p', '--password', help='password for admin')
3034    p.set_defaults(func=bdev_nvme_opal_init)
3035
3036    def bdev_nvme_opal_revert(args):
3037        rpc.nvme.bdev_nvme_opal_revert(args.client,
3038                                       nvme_ctrlr_name=args.nvme_ctrlr_name,
3039                                       password=args.password)
3040    p = subparsers.add_parser('bdev_nvme_opal_revert', help='Revert to default factory settings')
3041    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name')
3042    p.add_argument('-p', '--password', help='password')
3043    p.set_defaults(func=bdev_nvme_opal_revert)
3044
3045    def bdev_opal_create(args):
3046        print_json(rpc.bdev.bdev_opal_create(args.client,
3047                                             nvme_ctrlr_name=args.nvme_ctrlr_name,
3048                                             nsid=args.nsid,
3049                                             locking_range_id=args.locking_range_id,
3050                                             range_start=args.range_start,
3051                                             range_length=args.range_length,
3052                                             password=args.password))
3053
3054    p = subparsers.add_parser('bdev_opal_create', help="""Create opal bdev on specified NVMe controller""")
3055    p.add_argument('-b', '--nvme-ctrlr-name', help='nvme ctrlr name', required=True)
3056    p.add_argument('-n', '--nsid', help='namespace ID (only support nsid=1 for now)', type=int, required=True)
3057    p.add_argument('-i', '--locking-range-id', help='locking range id', type=int, required=True)
3058    p.add_argument('-s', '--range-start', help='locking range start LBA', type=int, required=True)
3059    p.add_argument('-l', '--range-length', help='locking range length (in blocks)', type=int, required=True)
3060    p.add_argument('-p', '--password', help='admin password', required=True)
3061    p.set_defaults(func=bdev_opal_create)
3062
3063    def bdev_opal_get_info(args):
3064        print_dict(rpc.bdev.bdev_opal_get_info(args.client,
3065                                               bdev_name=args.bdev_name,
3066                                               password=args.password))
3067
3068    p = subparsers.add_parser('bdev_opal_get_info', help='get opal locking range info for this bdev')
3069    p.add_argument('-b', '--bdev-name', help='opal bdev')
3070    p.add_argument('-p', '--password', help='password')
3071    p.set_defaults(func=bdev_opal_get_info)
3072
3073    def bdev_opal_delete(args):
3074        rpc.bdev.bdev_opal_delete(args.client,
3075                                  bdev_name=args.bdev_name,
3076                                  password=args.password)
3077
3078    p = subparsers.add_parser('bdev_opal_delete', help="""delete a virtual opal bdev""")
3079    p.add_argument('-b', '--bdev-name', help='opal virtual bdev', required=True)
3080    p.add_argument('-p', '--password', help='admin password', required=True)
3081    p.set_defaults(func=bdev_opal_delete)
3082
3083    def bdev_opal_new_user(args):
3084        rpc.bdev.bdev_opal_new_user(args.client,
3085                                    bdev_name=args.bdev_name,
3086                                    admin_password=args.admin_password,
3087                                    user_id=args.user_id,
3088                                    user_password=args.user_password)
3089
3090    p = subparsers.add_parser('bdev_opal_new_user', help="""Add a user to opal bdev who can set lock state for this bdev""")
3091    p.add_argument('-b', '--bdev-name', help='opal bdev', required=True)
3092    p.add_argument('-p', '--admin-password', help='admin password', required=True)
3093    p.add_argument('-i', '--user-id', help='ID for new user', type=int, required=True)
3094    p.add_argument('-u', '--user-password', help='password set for this user', required=True)
3095    p.set_defaults(func=bdev_opal_new_user)
3096
3097    def bdev_opal_set_lock_state(args):
3098        rpc.bdev.bdev_opal_set_lock_state(args.client,
3099                                          bdev_name=args.bdev_name,
3100                                          user_id=args.user_id,
3101                                          password=args.password,
3102                                          lock_state=args.lock_state)
3103
3104    p = subparsers.add_parser('bdev_opal_set_lock_state', help="""set lock state for an opal bdev""")
3105    p.add_argument('-b', '--bdev-name', help='opal bdev', required=True)
3106    p.add_argument('-i', '--user-id', help='ID of the user who want to set lock state, either admin or a user assigned to this bdev',
3107                   type=int, required=True)
3108    p.add_argument('-p', '--password', help='password of this user', required=True)
3109    p.add_argument('-l', '--lock-state', help='lock state to set, choose from {readwrite, readonly, rwlock}', required=True)
3110    p.set_defaults(func=bdev_opal_set_lock_state)
3111
3112    # bdev_nvme_send_cmd
3113    def bdev_nvme_send_cmd(args):
3114        print_dict(rpc.nvme.bdev_nvme_send_cmd(args.client,
3115                                               name=args.nvme_name,
3116                                               cmd_type=args.cmd_type,
3117                                               data_direction=args.data_direction,
3118                                               cmdbuf=args.cmdbuf,
3119                                               data=args.data,
3120                                               metadata=args.metadata,
3121                                               data_len=args.data_length,
3122                                               metadata_len=args.metadata_length,
3123                                               timeout_ms=args.timeout_ms))
3124
3125    p = subparsers.add_parser('bdev_nvme_send_cmd', help='NVMe passthrough cmd.')
3126    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""")
3127    p.add_argument('-t', '--cmd-type', help="""Type of nvme cmd. Valid values are: admin, io""")
3128    p.add_argument('-r', '--data-direction', help="""Direction of data transfer. Valid values are: c2h, h2c""")
3129    p.add_argument('-c', '--cmdbuf', help="""NVMe command encoded by base64 urlsafe""")
3130    p.add_argument('-d', '--data', help="""Data transferring to controller from host, encoded by base64 urlsafe""")
3131    p.add_argument('-m', '--metadata', help="""Metadata transferring to controller from host, encoded by base64 urlsafe""")
3132    p.add_argument('-D', '--data-length', help="""Data length required to transfer from controller to host""", type=int)
3133    p.add_argument('-M', '--metadata-length', help="""Metadata length required to transfer from controller to host""", type=int)
3134    p.add_argument('-T', '--timeout-ms',
3135                   help="""Command execution timeout value, in milliseconds,  if 0, don't track timeout""", type=int)
3136    p.set_defaults(func=bdev_nvme_send_cmd)
3137
3138    # Notifications
3139    def notify_get_types(args):
3140        print_dict(rpc.notify.notify_get_types(args.client))
3141
3142    p = subparsers.add_parser('notify_get_types', help='List available notifications that user can subscribe to.')
3143    p.set_defaults(func=notify_get_types)
3144
3145    def notify_get_notifications(args):
3146        ret = rpc.notify.notify_get_notifications(args.client,
3147                                                  id=args.id,
3148                                                  max=args.max)
3149        print_dict(ret)
3150
3151    p = subparsers.add_parser('notify_get_notifications', help='Get notifications')
3152    p.add_argument('-i', '--id', help="""First ID to start fetching from""", type=int)
3153    p.add_argument('-n', '--max', help="""Maximum number of notifications to return in response""", type=int)
3154    p.set_defaults(func=notify_get_notifications)
3155
3156    def thread_get_stats(args):
3157        print_dict(rpc.app.thread_get_stats(args.client))
3158
3159    p = subparsers.add_parser(
3160        'thread_get_stats', help='Display current statistics of all the threads')
3161    p.set_defaults(func=thread_get_stats)
3162
3163    def thread_set_cpumask(args):
3164        ret = rpc.app.thread_set_cpumask(args.client,
3165                                         id=args.id,
3166                                         cpumask=args.cpumask)
3167    p = subparsers.add_parser('thread_set_cpumask',
3168                              help="""set the cpumask of the thread whose ID matches to the
3169    specified value. The thread may be migrated to one of the specified CPUs.""")
3170    p.add_argument('-i', '--id', type=int, help='thread ID')
3171    p.add_argument('-m', '--cpumask', help='cpumask for this thread')
3172    p.set_defaults(func=thread_set_cpumask)
3173
3174    def log_enable_timestamps(args):
3175        ret = rpc.app.log_enable_timestamps(args.client,
3176                                            enabled=args.enabled)
3177    p = subparsers.add_parser('log_enable_timestamps',
3178                              help='Enable or disable timestamps.')
3179    p.add_argument('-d', '--disable', dest='enabled', default=False, action='store_false', help="Disable timestamps")
3180    p.add_argument('-e', '--enable', dest='enabled', action='store_true', help="Enable timestamps")
3181    p.set_defaults(func=log_enable_timestamps)
3182
3183    def thread_get_pollers(args):
3184        print_dict(rpc.app.thread_get_pollers(args.client))
3185
3186    p = subparsers.add_parser(
3187        'thread_get_pollers', help='Display current pollers of all the threads')
3188    p.set_defaults(func=thread_get_pollers)
3189
3190    def thread_get_io_channels(args):
3191        print_dict(rpc.app.thread_get_io_channels(args.client))
3192
3193    p = subparsers.add_parser(
3194        'thread_get_io_channels', help='Display current IO channels of all the threads')
3195    p.set_defaults(func=thread_get_io_channels)
3196
3197    def env_dpdk_get_mem_stats(args):
3198        print_dict(rpc.env_dpdk.env_dpdk_get_mem_stats(args.client))
3199
3200    p = subparsers.add_parser(
3201        'env_dpdk_get_mem_stats', help='write the dpdk memory stats to a file.')
3202    p.set_defaults(func=env_dpdk_get_mem_stats)
3203
3204    # blobfs
3205    def blobfs_detect(args):
3206        print(rpc.blobfs.blobfs_detect(args.client,
3207                                       bdev_name=args.bdev_name))
3208
3209    p = subparsers.add_parser('blobfs_detect', help='Detect whether a blobfs exists on bdev')
3210    p.add_argument('bdev_name', help='Blockdev name to detect blobfs. Example: Malloc0.')
3211    p.set_defaults(func=blobfs_detect)
3212
3213    def blobfs_create(args):
3214        print(rpc.blobfs.blobfs_create(args.client,
3215                                       bdev_name=args.bdev_name,
3216                                       cluster_sz=args.cluster_sz))
3217
3218    p = subparsers.add_parser('blobfs_create', help='Build a blobfs on bdev')
3219    p.add_argument('bdev_name', help='Blockdev name to build blobfs. Example: Malloc0.')
3220    p.add_argument('-c', '--cluster-sz',
3221                   help="""Size of cluster in bytes (Optional). Must be multiple of 4KB page size. Default and minimal value is 1M.""")
3222    p.set_defaults(func=blobfs_create)
3223
3224    def blobfs_mount(args):
3225        print(rpc.blobfs.blobfs_mount(args.client,
3226                                      bdev_name=args.bdev_name,
3227                                      mountpoint=args.mountpoint))
3228
3229    p = subparsers.add_parser('blobfs_mount', help='Mount a blobfs on bdev to host path by FUSE')
3230    p.add_argument('bdev_name', help='Blockdev name where the blobfs is. Example: Malloc0.')
3231    p.add_argument('mountpoint', help='Mountpoint path in host to mount blobfs. Example: /mnt/.')
3232    p.set_defaults(func=blobfs_mount)
3233
3234    def blobfs_set_cache_size(args):
3235        print(rpc.blobfs.blobfs_set_cache_size(args.client,
3236                                               size_in_mb=args.size_in_mb))
3237
3238    p = subparsers.add_parser('blobfs_set_cache_size', help='Set cache size for blobfs')
3239    p.add_argument('size_in_mb', help='Cache size for blobfs in megabytes.', type=int)
3240    p.set_defaults(func=blobfs_set_cache_size)
3241
3242    # sock
3243    def sock_impl_get_options(args):
3244        print_json(rpc.sock.sock_impl_get_options(args.client,
3245                                                  impl_name=args.impl))
3246
3247    p = subparsers.add_parser('sock_impl_get_options', help="""Get options of socket layer implementation""")
3248    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3249    p.set_defaults(func=sock_impl_get_options)
3250
3251    def sock_impl_set_options(args):
3252        rpc.sock.sock_impl_set_options(args.client,
3253                                       impl_name=args.impl,
3254                                       recv_buf_size=args.recv_buf_size,
3255                                       send_buf_size=args.send_buf_size,
3256                                       enable_recv_pipe=args.enable_recv_pipe,
3257                                       enable_quickack=args.enable_quickack,
3258                                       enable_placement_id=args.enable_placement_id,
3259                                       enable_zerocopy_send_server=args.enable_zerocopy_send_server,
3260                                       enable_zerocopy_send_client=args.enable_zerocopy_send_client,
3261                                       zerocopy_threshold=args.zerocopy_threshold,
3262                                       tls_version=args.tls_version,
3263                                       enable_ktls=args.enable_ktls)
3264
3265    p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""")
3266    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3267    p.add_argument('-r', '--recv-buf-size', help='Size of receive buffer on socket in bytes', type=int)
3268    p.add_argument('-s', '--send-buf-size', help='Size of send buffer on socket in bytes', type=int)
3269    p.add_argument('-p', '--enable-placement-id', help='Option for placement-id. 0:disable,1:incoming_napi,2:incoming_cpu', type=int)
3270    p.add_argument('--enable-recv-pipe', help='Enable receive pipe',
3271                   action='store_true', dest='enable_recv_pipe')
3272    p.add_argument('--disable-recv-pipe', help='Disable receive pipe',
3273                   action='store_false', dest='enable_recv_pipe')
3274    p.add_argument('--enable-quickack', help='Enable quick ACK',
3275                   action='store_true', dest='enable_quickack')
3276    p.add_argument('--disable-quickack', help='Disable quick ACK',
3277                   action='store_false', dest='enable_quickack')
3278    p.add_argument('--enable-zerocopy-send-server', help='Enable zerocopy on send for server sockets',
3279                   action='store_true', dest='enable_zerocopy_send_server')
3280    p.add_argument('--disable-zerocopy-send-server', help='Disable zerocopy on send for server sockets',
3281                   action='store_false', dest='enable_zerocopy_send_server')
3282    p.add_argument('--enable-zerocopy-send-client', help='Enable zerocopy on send for client sockets',
3283                   action='store_true', dest='enable_zerocopy_send_client')
3284    p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets',
3285                   action='store_false', dest='enable_zerocopy_send_client')
3286    p.add_argument('--zerocopy-threshold', help='Set zerocopy_threshold in bytes', type=int)
3287    p.add_argument('--tls-version', help='TLS protocol version', type=int)
3288    p.add_argument('--enable-ktls', help='Enable Kernel TLS',
3289                   action='store_true', dest='enable_ktls')
3290    p.add_argument('--disable-ktls', help='Disable Kernel TLS',
3291                   action='store_false', dest='enable_ktls')
3292    p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_quickack=None,
3293                   enable_placement_id=None, enable_zerocopy_send_server=None, enable_zerocopy_send_client=None,
3294                   zerocopy_threshold=None, tls_version=None, enable_ktls=None)
3295
3296    def sock_set_default_impl(args):
3297        print_json(rpc.sock.sock_set_default_impl(args.client,
3298                                                  impl_name=args.impl))
3299
3300    p = subparsers.add_parser('sock_set_default_impl', help="""Set the default sock implementation""")
3301    p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
3302    p.set_defaults(func=sock_set_default_impl)
3303
3304    def framework_get_pci_devices(args):
3305        def splitbuf(buf, step):
3306            return [buf[i:i+step] for i in range(0, len(buf), step)]
3307
3308        devices = rpc.subsystem.framework_get_pci_devices(args.client)
3309        if not args.format_lspci:
3310            print_json(devices)
3311        else:
3312            for devid, dev in enumerate(devices):
3313                print('{} device #{}'.format(dev['address'], devid))
3314                for lineid, line in enumerate(splitbuf(dev['config_space'], 32)):
3315                    print('{:02x}: {}'.format(lineid * 16, ' '.join(splitbuf(line.lower(), 2))))
3316                print()
3317
3318    p = subparsers.add_parser('framework_get_pci_devices', help='''Get a list of attached PCI devices''')
3319    p.add_argument('--format-lspci', help='Format the output in a way to be consumed by lspci -F',
3320                   action='store_true')
3321    p.set_defaults(func=framework_get_pci_devices)
3322
3323    # bdev_nvme_add_error_injection
3324    def bdev_nvme_add_error_injection(args):
3325        print_dict(rpc.nvme.bdev_nvme_add_error_injection(args.client,
3326                                                          name=args.nvme_name,
3327                                                          cmd_type=args.cmd_type,
3328                                                          opc=args.opc,
3329                                                          do_not_submit=args.do_not_submit,
3330                                                          timeout_in_us=args.timeout_in_us,
3331                                                          err_count=args.err_count,
3332                                                          sct=args.sct,
3333                                                          sc=args.sc))
3334    p = subparsers.add_parser('bdev_nvme_add_error_injection',
3335                              help='Add a NVMe command error injection.')
3336    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""", required=True)
3337    p.add_argument('-t', '--cmd-type', help="""Type of NVMe command. Valid values are: admin, io""", required=True)
3338    p.add_argument('-o', '--opc', help="""Opcode of the NVMe command.""", required=True, type=int)
3339    p.add_argument('-s', '--do-not-submit',
3340                   help="""Set to true if request should not be submitted to the controller""",
3341                   dest="do_not_submit", action='store_true')
3342    p.add_argument('-w', '--timeout-in-us', help="""Wait specified microseconds when do_not_submit is true""", type=int)
3343    p.add_argument('-e', '--err-count', help="""Number of matching NVMe commands to inject errors""", type=int)
3344    p.add_argument('-u', '--sct', help="""Status code type""", type=int)
3345    p.add_argument('-c', '--sc', help="""Status code""", type=int)
3346    p.set_defaults(func=bdev_nvme_add_error_injection)
3347
3348    # bdev_nvme_remove_error_injection
3349    def bdev_nvme_remove_error_injection(args):
3350        print_dict(rpc.nvme.bdev_nvme_remove_error_injection(args.client,
3351                                                             name=args.nvme_name,
3352                                                             cmd_type=args.cmd_type,
3353                                                             opc=args.opc))
3354    p = subparsers.add_parser('bdev_nvme_remove_error_injection',
3355                              help='Removes a NVMe command error injection.')
3356    p.add_argument('-n', '--nvme-name', help="""Name of the operating NVMe controller""", required=True)
3357    p.add_argument('-t', '--cmd-type', help="""Type of nvme cmd. Valid values are: admin, io""", required=True)
3358    p.add_argument('-o', '--opc', help="""Opcode of the nvme cmd.""", required=True, type=int)
3359    p.set_defaults(func=bdev_nvme_remove_error_injection)
3360
3361    def bdev_daos_create(args):
3362        num_blocks = (args.total_size * 1024 * 1024) // args.block_size
3363        print_json(rpc.bdev.bdev_daos_create(args.client,
3364                                             num_blocks=int(num_blocks),
3365                                             block_size=args.block_size,
3366                                             name=args.name,
3367                                             uuid=args.uuid,
3368                                             pool=args.pool,
3369                                             cont=args.cont,
3370                                             oclass=args.oclass))
3371    p = subparsers.add_parser('bdev_daos_create',
3372                              help='Create a bdev with DAOS backend')
3373    p.add_argument('name', help="Name of the bdev")
3374    p.add_argument('pool', help="UUID of the DAOS pool")
3375    p.add_argument('cont', help="UUID of the DAOS container")
3376    p.add_argument(
3377        'total_size', help='Size of DAOS bdev in MB (float > 0)', type=float)
3378    p.add_argument('block_size', help='Block size for this bdev', type=int)
3379    p.add_argument('-u', '--uuid', help="UUID of the bdev")
3380    p.add_argument('-o', '--oclass', help="DAOS object class")
3381    p.set_defaults(func=bdev_daos_create)
3382
3383    def bdev_daos_delete(args):
3384        rpc.bdev.bdev_daos_delete(args.client,
3385                                  name=args.name)
3386
3387    p = subparsers.add_parser('bdev_daos_delete',
3388                              help='Delete a DAOS disk')
3389    p.add_argument('name', help='DAOS bdev name')
3390    p.set_defaults(func=bdev_daos_delete)
3391
3392    def bdev_daos_resize(args):
3393        print_json(rpc.bdev.bdev_daos_resize(args.client,
3394                                             name=args.name,
3395                                             new_size=int(args.new_size)))
3396
3397    p = subparsers.add_parser('bdev_daos_resize',
3398                              help='Resize a DAOS bdev')
3399    p.add_argument('name', help='DAOS bdev name')
3400    p.add_argument('new_size', help='new bdev size for resize operation. The unit is MiB')
3401    p.set_defaults(func=bdev_daos_resize)
3402
3403    def iobuf_set_options(args):
3404        rpc.iobuf.iobuf_set_options(args.client,
3405                                    small_pool_count=args.small_pool_count,
3406                                    large_pool_count=args.large_pool_count,
3407                                    small_bufsize=args.small_bufsize,
3408                                    large_bufsize=args.large_bufsize)
3409    p = subparsers.add_parser('iobuf_set_options', help='Set iobuf pool options')
3410    p.add_argument('--small-pool-count', help='number of small buffers in the global pool', type=int)
3411    p.add_argument('--large-pool-count', help='number of large buffers in the global pool', type=int)
3412    p.add_argument('--small-bufsize', help='size of a small buffer', type=int)
3413    p.add_argument('--large-bufsize', help='size of a large buffer', type=int)
3414    p.set_defaults(func=iobuf_set_options)
3415
3416    def bdev_nvme_start_mdns_discovery(args):
3417        rpc.bdev.bdev_nvme_start_mdns_discovery(args.client,
3418                                                name=args.name,
3419                                                svcname=args.svcname,
3420                                                hostnqn=args.hostnqn)
3421
3422    p = subparsers.add_parser('bdev_nvme_start_mdns_discovery', help='Start mdns based automatic discovery')
3423    p.add_argument('-b', '--name', help="Name of the NVMe controller prefix for each bdev name", required=True)
3424    p.add_argument('-s', '--svcname', help='Service type to discover: e.g., _nvme-disc._tcp', required=True)
3425    p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn')
3426    p.set_defaults(func=bdev_nvme_start_mdns_discovery)
3427
3428    def bdev_nvme_stop_mdns_discovery(args):
3429        rpc.bdev.bdev_nvme_stop_mdns_discovery(args.client, name=args.name)
3430
3431    p = subparsers.add_parser('bdev_nvme_stop_mdns_discovery', help='Stop automatic mdns discovery')
3432    p.add_argument('-b', '--name', help="Name of the service to stop", required=True)
3433    p.set_defaults(func=bdev_nvme_stop_mdns_discovery)
3434
3435    def bdev_nvme_get_mdns_discovery_info(args):
3436        print_dict(rpc.bdev.bdev_nvme_get_mdns_discovery_info(args.client))
3437
3438    p = subparsers.add_parser('bdev_nvme_get_mdns_discovery_info', help='Get information about the automatic mdns discovery')
3439    p.set_defaults(func=bdev_nvme_get_mdns_discovery_info)
3440
3441    def check_called_name(name):
3442        if name in deprecated_aliases:
3443            print("{} is deprecated, use {} instead.".format(name, deprecated_aliases[name]), file=sys.stderr)
3444
3445    class dry_run_client:
3446        def call(self, method, params=None):
3447            print("Request:\n" + json.dumps({"method": method, "params": params}, indent=2))
3448
3449    def null_print(arg):
3450        pass
3451
3452    def call_rpc_func(args):
3453        args.func(args)
3454        check_called_name(args.called_rpc_name)
3455
3456    def execute_script(parser, client, fd):
3457        executed_rpc = ""
3458        for rpc_call in map(str.rstrip, fd):
3459            if not rpc_call.strip():
3460                continue
3461            executed_rpc = "\n".join([executed_rpc, rpc_call])
3462            rpc_args = shlex.split(rpc_call)
3463            if rpc_args[0][0] == '#':
3464                # Ignore lines starting with # - treat them as comments
3465                continue
3466            args = parser.parse_args(rpc_args)
3467            args.client = client
3468            try:
3469                call_rpc_func(args)
3470            except JSONRPCException as ex:
3471                print("Exception:")
3472                print(executed_rpc.strip() + " <<<")
3473                print(ex.message)
3474                exit(1)
3475
3476    def load_plugin(args):
3477        # Create temporary parser, pull out the plugin parameter, load the module, and then run the real argument parser
3478        plugin_parser = argparse.ArgumentParser(add_help=False)
3479        plugin_parser.add_argument('--plugin', dest='rpc_plugin', help='Module name of plugin with additional RPC commands')
3480
3481        rpc_module = plugin_parser.parse_known_args()[0].rpc_plugin
3482        if args is not None:
3483            rpc_module = plugin_parser.parse_known_args(args)[0].rpc_plugin
3484
3485        if rpc_module in plugins:
3486            return
3487
3488        if rpc_module is not None:
3489            try:
3490                rpc_plugin = importlib.import_module(rpc_module)
3491                try:
3492                    rpc_plugin.spdk_rpc_plugin_initialize(subparsers)
3493                    plugins.append(rpc_module)
3494                except AttributeError:
3495                    print("Module %s does not contain 'spdk_rpc_plugin_initialize' function" % rpc_module)
3496            except ModuleNotFoundError:
3497                print("Module %s not found" % rpc_module)
3498
3499    def replace_arg_underscores(args):
3500        # All option names are defined with dashes only - for example: --tgt-name
3501        # But if user used underscores, convert them to dashes (--tgt_name => --tgt-name)
3502        # SPDK was inconsistent previously and had some options with underscores, so
3503        # doing this conversion ensures backward compatibility with older scripts.
3504        for i in range(len(args)):
3505            arg = args[i]
3506            if arg.startswith('--') and "_" in arg:
3507                opt, *vals = arg.split('=')
3508                args[i] = '='.join([opt.replace('_', '-'), *vals])
3509
3510    plugins = []
3511    load_plugin(None)
3512
3513    replace_arg_underscores(sys.argv)
3514
3515    args = parser.parse_args()
3516
3517    if sys.stdin.isatty() and not hasattr(args, 'func'):
3518        # No arguments and no data piped through stdin
3519        parser.print_help()
3520        exit(1)
3521    if args.is_server:
3522        for input in sys.stdin:
3523            cmd = shlex.split(input)
3524            replace_arg_underscores(cmd)
3525            try:
3526                load_plugin(cmd)
3527                tmp_args = parser.parse_args(cmd)
3528            except SystemExit as ex:
3529                print("**STATUS=1", flush=True)
3530                continue
3531
3532            try:
3533                tmp_args.client = rpc.client.JSONRPCClient(
3534                    tmp_args.server_addr, tmp_args.port, tmp_args.timeout,
3535                    log_level=getattr(logging, tmp_args.verbose.upper()), conn_retries=tmp_args.conn_retries)
3536                call_rpc_func(tmp_args)
3537                print("**STATUS=0", flush=True)
3538            except JSONRPCException as ex:
3539                print(ex.message)
3540                print("**STATUS=1", flush=True)
3541        exit(0)
3542    elif args.dry_run:
3543        args.client = dry_run_client()
3544        print_dict = null_print
3545        print_json = null_print
3546        print_array = null_print
3547    else:
3548        try:
3549            args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.timeout,
3550                                                   log_level=getattr(logging, args.verbose.upper()),
3551                                                   conn_retries=args.conn_retries)
3552        except JSONRPCException as ex:
3553            print(ex.message)
3554            exit(1)
3555
3556    if hasattr(args, 'func'):
3557        try:
3558            call_rpc_func(args)
3559        except JSONRPCException as ex:
3560            print(ex.message)
3561            exit(1)
3562    else:
3563        execute_script(parser, args.client, sys.stdin)
3564