1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef __NVMF_INTERNAL_H__ 35 #define __NVMF_INTERNAL_H__ 36 37 #include "spdk/stdinc.h" 38 39 #include "spdk/likely.h" 40 #include "spdk/nvmf.h" 41 #include "spdk/nvmf_spec.h" 42 #include "spdk/assert.h" 43 #include "spdk/bdev.h" 44 #include "spdk/queue.h" 45 #include "spdk/util.h" 46 #include "spdk/thread.h" 47 48 #define SPDK_NVMF_MAX_SGL_ENTRIES 16 49 50 /* AIO backend requires block size aligned data buffers, 51 * extra 4KiB aligned data buffer should work for most devices. 52 */ 53 #define SHIFT_4KB 12u 54 #define NVMF_DATA_BUFFER_ALIGNMENT (1u << SHIFT_4KB) 55 #define NVMF_DATA_BUFFER_MASK (NVMF_DATA_BUFFER_ALIGNMENT - 1LL) 56 57 enum spdk_nvmf_subsystem_state { 58 SPDK_NVMF_SUBSYSTEM_INACTIVE = 0, 59 SPDK_NVMF_SUBSYSTEM_ACTIVATING, 60 SPDK_NVMF_SUBSYSTEM_ACTIVE, 61 SPDK_NVMF_SUBSYSTEM_PAUSING, 62 SPDK_NVMF_SUBSYSTEM_PAUSED, 63 SPDK_NVMF_SUBSYSTEM_RESUMING, 64 SPDK_NVMF_SUBSYSTEM_DEACTIVATING, 65 }; 66 67 enum spdk_nvmf_qpair_state { 68 SPDK_NVMF_QPAIR_UNINITIALIZED = 0, 69 SPDK_NVMF_QPAIR_ACTIVATING, 70 SPDK_NVMF_QPAIR_ACTIVE, 71 SPDK_NVMF_QPAIR_DEACTIVATING, 72 SPDK_NVMF_QPAIR_ERROR, 73 }; 74 75 typedef void (*spdk_nvmf_state_change_done)(void *cb_arg, int status); 76 77 struct spdk_nvmf_tgt { 78 uint64_t discovery_genctr; 79 80 uint32_t max_subsystems; 81 82 /* Array of subsystem pointers of size max_subsystems indexed by sid */ 83 struct spdk_nvmf_subsystem **subsystems; 84 85 struct spdk_nvmf_discovery_log_page *discovery_log_page; 86 size_t discovery_log_page_size; 87 TAILQ_HEAD(, spdk_nvmf_transport) transports; 88 89 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn; 90 void *destroy_cb_arg; 91 }; 92 93 struct spdk_nvmf_host { 94 char *nqn; 95 TAILQ_ENTRY(spdk_nvmf_host) link; 96 }; 97 98 struct spdk_nvmf_listener { 99 struct spdk_nvme_transport_id trid; 100 struct spdk_nvmf_transport *transport; 101 TAILQ_ENTRY(spdk_nvmf_listener) link; 102 }; 103 104 struct spdk_nvmf_transport_pg_cache_buf { 105 STAILQ_ENTRY(spdk_nvmf_transport_pg_cache_buf) link; 106 }; 107 108 struct spdk_nvmf_transport_poll_group { 109 struct spdk_nvmf_transport *transport; 110 STAILQ_HEAD(, spdk_nvmf_transport_pg_cache_buf) buf_cache; 111 uint32_t buf_cache_count; 112 uint32_t buf_cache_size; 113 TAILQ_ENTRY(spdk_nvmf_transport_poll_group) link; 114 }; 115 116 struct spdk_nvmf_subsystem_poll_group { 117 /* Array of channels for each namespace indexed by nsid - 1 */ 118 struct spdk_io_channel **channels; 119 uint32_t num_channels; 120 121 enum spdk_nvmf_subsystem_state state; 122 123 TAILQ_HEAD(, spdk_nvmf_request) queued; 124 }; 125 126 struct spdk_nvmf_poll_group { 127 struct spdk_thread *thread; 128 struct spdk_poller *poller; 129 130 TAILQ_HEAD(, spdk_nvmf_transport_poll_group) tgroups; 131 132 /* Array of poll groups indexed by subsystem id (sid) */ 133 struct spdk_nvmf_subsystem_poll_group *sgroups; 134 uint32_t num_sgroups; 135 136 /* All of the queue pairs that belong to this poll group */ 137 TAILQ_HEAD(, spdk_nvmf_qpair) qpairs; 138 }; 139 140 typedef enum _spdk_nvmf_request_exec_status { 141 SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE, 142 SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS, 143 } spdk_nvmf_request_exec_status; 144 145 union nvmf_h2c_msg { 146 struct spdk_nvmf_capsule_cmd nvmf_cmd; 147 struct spdk_nvme_cmd nvme_cmd; 148 struct spdk_nvmf_fabric_prop_set_cmd prop_set_cmd; 149 struct spdk_nvmf_fabric_prop_get_cmd prop_get_cmd; 150 struct spdk_nvmf_fabric_connect_cmd connect_cmd; 151 }; 152 SPDK_STATIC_ASSERT(sizeof(union nvmf_h2c_msg) == 64, "Incorrect size"); 153 154 union nvmf_c2h_msg { 155 struct spdk_nvme_cpl nvme_cpl; 156 struct spdk_nvmf_fabric_prop_get_rsp prop_get_rsp; 157 struct spdk_nvmf_fabric_connect_rsp connect_rsp; 158 }; 159 SPDK_STATIC_ASSERT(sizeof(union nvmf_c2h_msg) == 16, "Incorrect size"); 160 161 struct spdk_nvmf_request { 162 struct spdk_nvmf_qpair *qpair; 163 uint32_t length; 164 enum spdk_nvme_data_transfer xfer; 165 void *data; 166 union nvmf_h2c_msg *cmd; 167 union nvmf_c2h_msg *rsp; 168 struct iovec iov[SPDK_NVMF_MAX_SGL_ENTRIES * 2]; 169 uint32_t iovcnt; 170 struct spdk_bdev_io_wait_entry bdev_io_wait; 171 172 TAILQ_ENTRY(spdk_nvmf_request) link; 173 }; 174 175 struct spdk_nvmf_ns { 176 uint32_t nsid; 177 struct spdk_nvmf_subsystem *subsystem; 178 struct spdk_bdev *bdev; 179 struct spdk_bdev_desc *desc; 180 struct spdk_nvmf_ns_opts opts; 181 }; 182 183 struct spdk_nvmf_qpair { 184 enum spdk_nvmf_qpair_state state; 185 spdk_nvmf_state_change_done state_cb; 186 void *state_cb_arg; 187 188 struct spdk_nvmf_transport *transport; 189 struct spdk_nvmf_ctrlr *ctrlr; 190 struct spdk_nvmf_poll_group *group; 191 192 uint16_t qid; 193 uint16_t sq_head; 194 uint16_t sq_head_max; 195 196 TAILQ_HEAD(, spdk_nvmf_request) outstanding; 197 TAILQ_ENTRY(spdk_nvmf_qpair) link; 198 }; 199 200 struct spdk_nvmf_ctrlr_feat { 201 union spdk_nvme_feat_arbitration arbitration; 202 union spdk_nvme_feat_power_management power_management; 203 union spdk_nvme_feat_error_recovery error_recovery; 204 union spdk_nvme_feat_volatile_write_cache volatile_write_cache; 205 union spdk_nvme_feat_number_of_queues number_of_queues; 206 union spdk_nvme_feat_write_atomicity write_atomicity; 207 union spdk_nvme_feat_async_event_configuration async_event_configuration; 208 union spdk_nvme_feat_keep_alive_timer keep_alive_timer; 209 }; 210 211 /* 212 * This structure represents an NVMe-oF controller, 213 * which is like a "session" in networking terms. 214 */ 215 struct spdk_nvmf_ctrlr { 216 uint16_t cntlid; 217 struct spdk_nvmf_subsystem *subsys; 218 219 struct { 220 union spdk_nvme_cap_register cap; 221 union spdk_nvme_vs_register vs; 222 union spdk_nvme_cc_register cc; 223 union spdk_nvme_csts_register csts; 224 } vcprop; /* virtual controller properties */ 225 226 struct spdk_nvmf_ctrlr_feat feat; 227 228 struct spdk_nvmf_qpair *admin_qpair; 229 struct spdk_thread *thread; 230 struct spdk_bit_array *qpair_mask; 231 232 struct spdk_nvmf_request *aer_req; 233 union spdk_nvme_async_event_completion notice_event; 234 struct spdk_uuid hostid; 235 236 uint16_t changed_ns_list_count; 237 struct spdk_nvme_ns_list changed_ns_list; 238 239 /* Time to trigger keep-alive--poller_time = now_tick + period */ 240 uint64_t last_keep_alive_tick; 241 struct spdk_poller *keep_alive_poller; 242 243 TAILQ_ENTRY(spdk_nvmf_ctrlr) link; 244 }; 245 246 struct spdk_nvmf_subsystem { 247 struct spdk_thread *thread; 248 uint32_t id; 249 enum spdk_nvmf_subsystem_state state; 250 251 char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; 252 enum spdk_nvmf_subtype subtype; 253 uint16_t next_cntlid; 254 bool allow_any_host; 255 256 struct spdk_nvmf_tgt *tgt; 257 258 char sn[SPDK_NVME_CTRLR_SN_LEN + 1]; 259 260 /* Array of pointers to namespaces of size max_nsid indexed by nsid - 1 */ 261 struct spdk_nvmf_ns **ns; 262 uint32_t max_nsid; 263 /* This is the maximum allowed nsid to a subsystem */ 264 uint32_t max_allowed_nsid; 265 266 TAILQ_HEAD(, spdk_nvmf_ctrlr) ctrlrs; 267 268 TAILQ_HEAD(, spdk_nvmf_host) hosts; 269 270 TAILQ_HEAD(, spdk_nvmf_listener) listeners; 271 272 TAILQ_ENTRY(spdk_nvmf_subsystem) entries; 273 }; 274 275 typedef void(*spdk_nvmf_poll_group_mod_done)(void *cb_arg, int status); 276 277 struct spdk_nvmf_transport *spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, 278 enum spdk_nvme_transport_type); 279 280 int spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 281 struct spdk_nvmf_transport *transport); 282 int spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 283 struct spdk_nvmf_subsystem *subsystem); 284 int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 285 struct spdk_nvmf_subsystem *subsystem, 286 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 287 void spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 288 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 289 void spdk_nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 290 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 291 void spdk_nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 292 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 293 void spdk_nvmf_request_exec(struct spdk_nvmf_request *req); 294 int spdk_nvmf_request_free(struct spdk_nvmf_request *req); 295 int spdk_nvmf_request_complete(struct spdk_nvmf_request *req); 296 297 void spdk_nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, struct iovec *iov, 298 uint32_t iovcnt, uint64_t offset, uint32_t length); 299 300 void spdk_nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr); 301 int spdk_nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req); 302 int spdk_nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req); 303 int spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req); 304 bool spdk_nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr); 305 bool spdk_nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr); 306 void spdk_nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid); 307 308 void spdk_nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata); 309 310 int spdk_nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem, 311 struct spdk_nvmf_ctrlr *ctrlr); 312 void spdk_nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem, 313 struct spdk_nvmf_ctrlr *ctrlr); 314 struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem, 315 uint16_t cntlid); 316 int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr); 317 318 /* 319 * Abort aer is sent on a per controller basis and sends a completion for the aer to the host. 320 * This function should be called when attempting to recover in error paths when it is OK for 321 * the host to send a subsequent AER. 322 */ 323 void spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr); 324 325 /* 326 * Free aer simply frees the rdma resources for the aer without informing the host. 327 * This function should be called when deleting a qpair when one wants to make sure 328 * the qpair is completely empty before freeing the request. The reason we free the 329 * AER without sending a completion is to prevent the host from sending another AER. 330 */ 331 void spdk_nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair); 332 333 static inline struct spdk_nvmf_ns * 334 _spdk_nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid) 335 { 336 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 337 if (spdk_unlikely(nsid - 1 >= subsystem->max_nsid)) { 338 return NULL; 339 } 340 341 return subsystem->ns[nsid - 1]; 342 } 343 344 static inline bool 345 spdk_nvmf_qpair_is_admin_queue(struct spdk_nvmf_qpair *qpair) 346 { 347 return qpair->qid == 0; 348 } 349 350 #endif /* __NVMF_INTERNAL_H__ */ 351