1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef __NVMF_INTERNAL_H__ 35 #define __NVMF_INTERNAL_H__ 36 37 #include "spdk/stdinc.h" 38 39 #include "spdk/likely.h" 40 #include "spdk/nvmf.h" 41 #include "spdk/nvmf_cmd.h" 42 #include "spdk/nvmf_transport.h" 43 #include "spdk/nvmf_spec.h" 44 #include "spdk/assert.h" 45 #include "spdk/bdev.h" 46 #include "spdk/queue.h" 47 #include "spdk/util.h" 48 #include "spdk/thread.h" 49 50 #define NVMF_MAX_ASYNC_EVENTS (4) 51 52 enum spdk_nvmf_subsystem_state { 53 SPDK_NVMF_SUBSYSTEM_INACTIVE = 0, 54 SPDK_NVMF_SUBSYSTEM_ACTIVATING, 55 SPDK_NVMF_SUBSYSTEM_ACTIVE, 56 SPDK_NVMF_SUBSYSTEM_PAUSING, 57 SPDK_NVMF_SUBSYSTEM_PAUSED, 58 SPDK_NVMF_SUBSYSTEM_RESUMING, 59 SPDK_NVMF_SUBSYSTEM_DEACTIVATING, 60 SPDK_NVMF_SUBSYSTEM_NUM_STATES, 61 }; 62 63 struct spdk_nvmf_tgt { 64 char name[NVMF_TGT_NAME_MAX_LENGTH]; 65 66 pthread_mutex_t mutex; 67 68 uint64_t discovery_genctr; 69 70 struct spdk_poller *accept_poller; 71 72 uint32_t max_subsystems; 73 74 /* Array of subsystem pointers of size max_subsystems indexed by sid */ 75 struct spdk_nvmf_subsystem **subsystems; 76 77 TAILQ_HEAD(, spdk_nvmf_transport) transports; 78 TAILQ_HEAD(, spdk_nvmf_poll_group) poll_groups; 79 80 /* Used for round-robin assignment of connections to poll groups */ 81 struct spdk_nvmf_poll_group *next_poll_group; 82 83 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn; 84 void *destroy_cb_arg; 85 86 TAILQ_ENTRY(spdk_nvmf_tgt) link; 87 }; 88 89 struct spdk_nvmf_host { 90 char nqn[SPDK_NVMF_NQN_MAX_LEN + 1]; 91 TAILQ_ENTRY(spdk_nvmf_host) link; 92 }; 93 94 struct spdk_nvmf_subsystem_listener { 95 struct spdk_nvmf_subsystem *subsystem; 96 spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn; 97 void *cb_arg; 98 struct spdk_nvme_transport_id *trid; 99 struct spdk_nvmf_transport *transport; 100 enum spdk_nvme_ana_state ana_state; 101 uint64_t ana_state_change_count; 102 TAILQ_ENTRY(spdk_nvmf_subsystem_listener) link; 103 }; 104 105 /* Maximum number of registrants supported per namespace */ 106 #define SPDK_NVMF_MAX_NUM_REGISTRANTS 16 107 108 struct spdk_nvmf_registrant_info { 109 uint64_t rkey; 110 char host_uuid[SPDK_UUID_STRING_LEN]; 111 }; 112 113 struct spdk_nvmf_reservation_info { 114 bool ptpl_activated; 115 enum spdk_nvme_reservation_type rtype; 116 uint64_t crkey; 117 char bdev_uuid[SPDK_UUID_STRING_LEN]; 118 char holder_uuid[SPDK_UUID_STRING_LEN]; 119 uint32_t num_regs; 120 struct spdk_nvmf_registrant_info registrants[SPDK_NVMF_MAX_NUM_REGISTRANTS]; 121 }; 122 123 struct spdk_nvmf_subsystem_pg_ns_info { 124 struct spdk_io_channel *channel; 125 struct spdk_uuid uuid; 126 /* current reservation key, no reservation if the value is 0 */ 127 uint64_t crkey; 128 /* reservation type */ 129 enum spdk_nvme_reservation_type rtype; 130 /* Host ID which holds the reservation */ 131 struct spdk_uuid holder_id; 132 /* Host ID for the registrants with the namespace */ 133 struct spdk_uuid reg_hostid[SPDK_NVMF_MAX_NUM_REGISTRANTS]; 134 uint64_t num_blocks; 135 }; 136 137 typedef void(*spdk_nvmf_poll_group_mod_done)(void *cb_arg, int status); 138 139 struct spdk_nvmf_subsystem_poll_group { 140 /* Array of namespace information for each namespace indexed by nsid - 1 */ 141 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 142 uint32_t num_ns; 143 144 uint64_t io_outstanding; 145 spdk_nvmf_poll_group_mod_done cb_fn; 146 void *cb_arg; 147 148 enum spdk_nvmf_subsystem_state state; 149 150 TAILQ_HEAD(, spdk_nvmf_request) queued; 151 }; 152 153 struct spdk_nvmf_registrant { 154 TAILQ_ENTRY(spdk_nvmf_registrant) link; 155 struct spdk_uuid hostid; 156 /* Registration key */ 157 uint64_t rkey; 158 }; 159 160 struct spdk_nvmf_ns { 161 uint32_t nsid; 162 struct spdk_nvmf_subsystem *subsystem; 163 struct spdk_bdev *bdev; 164 struct spdk_bdev_desc *desc; 165 struct spdk_nvmf_ns_opts opts; 166 /* reservation notificaton mask */ 167 uint32_t mask; 168 /* generation code */ 169 uint32_t gen; 170 /* registrants head */ 171 TAILQ_HEAD(, spdk_nvmf_registrant) registrants; 172 /* current reservation key */ 173 uint64_t crkey; 174 /* reservation type */ 175 enum spdk_nvme_reservation_type rtype; 176 /* current reservation holder, only valid if reservation type can only have one holder */ 177 struct spdk_nvmf_registrant *holder; 178 /* Persist Through Power Loss file which contains the persistent reservation */ 179 char *ptpl_file; 180 /* Persist Through Power Loss feature is enabled */ 181 bool ptpl_activated; 182 }; 183 184 struct spdk_nvmf_ctrlr_feat { 185 union spdk_nvme_feat_arbitration arbitration; 186 union spdk_nvme_feat_power_management power_management; 187 union spdk_nvme_feat_error_recovery error_recovery; 188 union spdk_nvme_feat_volatile_write_cache volatile_write_cache; 189 union spdk_nvme_feat_number_of_queues number_of_queues; 190 union spdk_nvme_feat_write_atomicity write_atomicity; 191 union spdk_nvme_feat_async_event_configuration async_event_configuration; 192 union spdk_nvme_feat_keep_alive_timer keep_alive_timer; 193 }; 194 195 /* 196 * NVMf reservation notificaton log page. 197 */ 198 struct spdk_nvmf_reservation_log { 199 struct spdk_nvme_reservation_notification_log log; 200 TAILQ_ENTRY(spdk_nvmf_reservation_log) link; 201 struct spdk_nvmf_ctrlr *ctrlr; 202 }; 203 204 /* 205 * NVMf async event completion. 206 */ 207 struct spdk_nvmf_async_event_completion { 208 union spdk_nvme_async_event_completion event; 209 STAILQ_ENTRY(spdk_nvmf_async_event_completion) link; 210 }; 211 212 /* 213 * This structure represents an NVMe-oF controller, 214 * which is like a "session" in networking terms. 215 */ 216 struct spdk_nvmf_ctrlr { 217 uint16_t cntlid; 218 char hostnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; 219 struct spdk_nvmf_subsystem *subsys; 220 221 struct spdk_nvmf_ctrlr_data cdata; 222 223 struct spdk_nvmf_registers vcprop; 224 225 struct spdk_nvmf_ctrlr_feat feat; 226 227 struct spdk_nvmf_qpair *admin_qpair; 228 struct spdk_thread *thread; 229 struct spdk_bit_array *qpair_mask; 230 231 const struct spdk_nvmf_subsystem_listener *listener; 232 233 struct spdk_nvmf_request *aer_req[NVMF_MAX_ASYNC_EVENTS]; 234 STAILQ_HEAD(, spdk_nvmf_async_event_completion) async_events; 235 uint64_t notice_aen_mask; 236 uint8_t nr_aer_reqs; 237 struct spdk_uuid hostid; 238 239 uint16_t changed_ns_list_count; 240 struct spdk_nvme_ns_list changed_ns_list; 241 uint64_t log_page_count; 242 uint8_t num_avail_log_pages; 243 TAILQ_HEAD(log_page_head, spdk_nvmf_reservation_log) log_head; 244 245 /* Time to trigger keep-alive--poller_time = now_tick + period */ 246 uint64_t last_keep_alive_tick; 247 struct spdk_poller *keep_alive_poller; 248 249 struct spdk_poller *association_timer; 250 251 bool dif_insert_or_strip; 252 bool in_destruct; 253 254 TAILQ_ENTRY(spdk_nvmf_ctrlr) link; 255 }; 256 257 struct spdk_nvmf_subsystem { 258 struct spdk_thread *thread; 259 260 uint32_t id; 261 262 enum spdk_nvmf_subsystem_state state; 263 enum spdk_nvmf_subtype subtype; 264 265 uint16_t next_cntlid; 266 struct { 267 uint8_t allow_any_host : 1; 268 uint8_t allow_any_listener : 1; 269 uint8_t ana_reporting : 1; 270 uint8_t reserved : 5; 271 } flags; 272 273 /* boolean for state change synchronization */ 274 bool changing_state; 275 276 struct spdk_nvmf_tgt *tgt; 277 278 /* Array of pointers to namespaces of size max_nsid indexed by nsid - 1 */ 279 struct spdk_nvmf_ns **ns; 280 uint32_t max_nsid; 281 282 TAILQ_HEAD(, spdk_nvmf_ctrlr) ctrlrs; 283 284 /* A mutex used to protect the hosts list and allow_any_host flag. Unlike the namespace 285 * array, this list is not used on the I/O path (it's needed for handling things like 286 * the CONNECT command), so use a mutex to protect it instead of requiring the subsystem 287 * state to be paused. This removes the requirement to pause the subsystem when hosts 288 * are added or removed dynamically. */ 289 pthread_mutex_t mutex; 290 TAILQ_HEAD(, spdk_nvmf_host) hosts; 291 TAILQ_HEAD(, spdk_nvmf_subsystem_listener) listeners; 292 293 TAILQ_ENTRY(spdk_nvmf_subsystem) entries; 294 295 char sn[SPDK_NVME_CTRLR_SN_LEN + 1]; 296 char mn[SPDK_NVME_CTRLR_MN_LEN + 1]; 297 char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; 298 }; 299 300 int nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 301 struct spdk_nvmf_transport *transport); 302 int nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 303 struct spdk_nvmf_subsystem *subsystem); 304 int nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 305 struct spdk_nvmf_subsystem *subsystem, 306 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 307 void nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 308 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 309 void nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 310 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 311 void nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 312 struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); 313 314 void nvmf_update_discovery_log(struct spdk_nvmf_tgt *tgt, const char *hostnqn); 315 void nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, const char *hostnqn, 316 struct iovec *iov, 317 uint32_t iovcnt, uint64_t offset, uint32_t length); 318 319 void nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr); 320 int nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req); 321 int nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req); 322 bool nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr); 323 bool nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr); 324 void nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid); 325 326 void nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata, 327 bool dif_insert_or_strip); 328 int nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 329 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 330 int nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 331 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 332 int nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 333 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 334 int nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 335 struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req); 336 int nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 337 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 338 int nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 339 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 340 int nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 341 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 342 int nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 343 struct spdk_io_channel *ch, struct spdk_nvmf_request *req); 344 bool nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev *bdev, struct spdk_nvme_cmd *cmd, 345 struct spdk_dif_ctx *dif_ctx); 346 347 int nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem, 348 struct spdk_nvmf_ctrlr *ctrlr); 349 void nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem, 350 struct spdk_nvmf_ctrlr *ctrlr); 351 void nvmf_subsystem_remove_all_listeners(struct spdk_nvmf_subsystem *subsystem, 352 bool stop); 353 struct spdk_nvmf_ctrlr *nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem, 354 uint16_t cntlid); 355 struct spdk_nvmf_subsystem_listener *nvmf_subsystem_find_listener( 356 struct spdk_nvmf_subsystem *subsystem, 357 const struct spdk_nvme_transport_id *trid); 358 struct spdk_nvmf_listener *nvmf_transport_find_listener( 359 struct spdk_nvmf_transport *transport, 360 const struct spdk_nvme_transport_id *trid); 361 void nvmf_subsystem_set_ana_state(struct spdk_nvmf_subsystem *subsystem, 362 const struct spdk_nvme_transport_id *trid, 363 enum spdk_nvme_ana_state ana_state, 364 spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn, void *cb_arg); 365 366 int nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr); 367 int nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr); 368 int nvmf_ctrlr_async_event_discovery_log_change_notice(struct spdk_nvmf_ctrlr *ctrlr); 369 void nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr); 370 void nvmf_ns_reservation_request(void *ctx); 371 void nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr, 372 struct spdk_nvmf_ns *ns, 373 enum spdk_nvme_reservation_notification_log_page_type type); 374 375 /* 376 * Abort aer is sent on a per controller basis and sends a completion for the aer to the host. 377 * This function should be called when attempting to recover in error paths when it is OK for 378 * the host to send a subsequent AER. 379 */ 380 void nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr); 381 382 /* 383 * Free aer simply frees the rdma resources for the aer without informing the host. 384 * This function should be called when deleting a qpair when one wants to make sure 385 * the qpair is completely empty before freeing the request. The reason we free the 386 * AER without sending a completion is to prevent the host from sending another AER. 387 */ 388 void nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair); 389 390 int nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req); 391 392 static inline struct spdk_nvmf_ns * 393 _nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid) 394 { 395 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 396 if (spdk_unlikely(nsid - 1 >= subsystem->max_nsid)) { 397 return NULL; 398 } 399 400 return subsystem->ns[nsid - 1]; 401 } 402 403 static inline bool 404 nvmf_qpair_is_admin_queue(struct spdk_nvmf_qpair *qpair) 405 { 406 return qpair->qid == 0; 407 } 408 409 #endif /* __NVMF_INTERNAL_H__ */ 410