1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef __NVMF_INTERNAL_H__ 35 #define __NVMF_INTERNAL_H__ 36 37 #include "spdk/stdinc.h" 38 39 #include "spdk/likely.h" 40 #include "spdk/nvmf.h" 41 #include "spdk/nvmf_spec.h" 42 #include "spdk/assert.h" 43 #include "spdk/queue.h" 44 #include "spdk/util.h" 45 46 #define SPDK_NVMF_DEFAULT_NUM_CTRLRS_PER_LCORE 1 47 48 enum spdk_nvmf_subsystem_state { 49 SPDK_NVMF_SUBSYSTEM_INACTIVE = 0, 50 SPDK_NVMF_SUBSYSTEM_ACTIVATING, 51 SPDK_NVMF_SUBSYSTEM_ACTIVE, 52 SPDK_NVMF_SUBSYSTEM_PAUSING, 53 SPDK_NVMF_SUBSYSTEM_PAUSED, 54 SPDK_NVMF_SUBSYSTEM_RESUMING, 55 SPDK_NVMF_SUBSYSTEM_DEACTIVATING, 56 }; 57 58 struct spdk_nvmf_tgt { 59 struct spdk_nvmf_tgt_opts opts; 60 61 uint64_t discovery_genctr; 62 63 /* Array of subsystem pointers of size max_subsystems indexed by sid */ 64 struct spdk_nvmf_subsystem **subsystems; 65 66 struct spdk_nvmf_discovery_log_page *discovery_log_page; 67 size_t discovery_log_page_size; 68 TAILQ_HEAD(, spdk_nvmf_transport) transports; 69 }; 70 71 struct spdk_nvmf_host { 72 char *nqn; 73 TAILQ_ENTRY(spdk_nvmf_host) link; 74 }; 75 76 struct spdk_nvmf_listener { 77 struct spdk_nvme_transport_id trid; 78 struct spdk_nvmf_transport *transport; 79 TAILQ_ENTRY(spdk_nvmf_listener) link; 80 }; 81 82 struct spdk_nvmf_transport_poll_group { 83 struct spdk_nvmf_transport *transport; 84 TAILQ_ENTRY(spdk_nvmf_transport_poll_group) link; 85 }; 86 87 struct spdk_nvmf_subsystem_poll_group { 88 /* Array of channels for each namespace indexed by nsid - 1 */ 89 struct spdk_io_channel **channels; 90 uint32_t num_channels; 91 92 enum spdk_nvmf_subsystem_state state; 93 94 TAILQ_HEAD(, spdk_nvmf_request) queued; 95 TAILQ_HEAD(, spdk_nvmf_request) outstanding; 96 }; 97 98 struct spdk_nvmf_poll_group { 99 struct spdk_thread *thread; 100 struct spdk_poller *poller; 101 102 TAILQ_HEAD(, spdk_nvmf_transport_poll_group) tgroups; 103 104 /* Array of poll groups indexed by subsystem id (sid) */ 105 struct spdk_nvmf_subsystem_poll_group *sgroups; 106 uint32_t num_sgroups; 107 }; 108 109 typedef enum _spdk_nvmf_request_exec_status { 110 SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE, 111 SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS, 112 } spdk_nvmf_request_exec_status; 113 114 union nvmf_h2c_msg { 115 struct spdk_nvmf_capsule_cmd nvmf_cmd; 116 struct spdk_nvme_cmd nvme_cmd; 117 struct spdk_nvmf_fabric_prop_set_cmd prop_set_cmd; 118 struct spdk_nvmf_fabric_prop_get_cmd prop_get_cmd; 119 struct spdk_nvmf_fabric_connect_cmd connect_cmd; 120 }; 121 SPDK_STATIC_ASSERT(sizeof(union nvmf_h2c_msg) == 64, "Incorrect size"); 122 123 union nvmf_c2h_msg { 124 struct spdk_nvme_cpl nvme_cpl; 125 struct spdk_nvmf_fabric_prop_get_rsp prop_get_rsp; 126 struct spdk_nvmf_fabric_connect_rsp connect_rsp; 127 }; 128 SPDK_STATIC_ASSERT(sizeof(union nvmf_c2h_msg) == 16, "Incorrect size"); 129 130 struct spdk_nvmf_request { 131 struct spdk_nvmf_qpair *qpair; 132 uint32_t length; 133 enum spdk_nvme_data_transfer xfer; 134 void *data; 135 union nvmf_h2c_msg *cmd; 136 union nvmf_c2h_msg *rsp; 137 138 TAILQ_ENTRY(spdk_nvmf_request) link; 139 }; 140 141 struct spdk_nvmf_ns { 142 struct spdk_nvmf_subsystem *subsystem; 143 struct spdk_bdev *bdev; 144 struct spdk_bdev_desc *desc; 145 struct spdk_nvmf_ns_opts opts; 146 }; 147 148 struct spdk_nvmf_qpair { 149 struct spdk_nvmf_transport *transport; 150 struct spdk_nvmf_ctrlr *ctrlr; 151 struct spdk_nvmf_poll_group *group; 152 153 uint16_t qid; 154 uint16_t sq_head; 155 uint16_t sq_head_max; 156 157 TAILQ_ENTRY(spdk_nvmf_qpair) link; 158 }; 159 160 struct spdk_nvmf_ctrlr_feat { 161 union spdk_nvme_feat_arbitration arbitration; 162 union spdk_nvme_feat_power_management power_management; 163 union spdk_nvme_feat_error_recovery error_recovery; 164 union spdk_nvme_feat_volatile_write_cache volatile_write_cache; 165 union spdk_nvme_feat_number_of_queues number_of_queues; 166 union spdk_nvme_feat_write_atomicity write_atomicity; 167 union spdk_nvme_feat_async_event_configuration async_event_configuration; 168 union spdk_nvme_feat_keep_alive_timer keep_alive_timer; 169 }; 170 171 /* 172 * This structure represents an NVMe-oF controller, 173 * which is like a "session" in networking terms. 174 */ 175 struct spdk_nvmf_ctrlr { 176 uint16_t cntlid; 177 struct spdk_nvmf_subsystem *subsys; 178 179 struct { 180 union spdk_nvme_cap_register cap; 181 union spdk_nvme_vs_register vs; 182 union spdk_nvme_cc_register cc; 183 union spdk_nvme_csts_register csts; 184 } vcprop; /* virtual controller properties */ 185 186 struct spdk_nvmf_ctrlr_feat feat; 187 188 struct spdk_nvmf_qpair *admin_qpair; 189 TAILQ_HEAD(, spdk_nvmf_qpair) qpairs; 190 int num_qpairs; 191 int max_qpairs_allowed; 192 struct spdk_nvmf_request *aer_req; 193 union spdk_nvme_async_event_completion notice_event; 194 uint8_t hostid[16]; 195 196 uint16_t changed_ns_list_count; 197 struct spdk_nvme_ns_list changed_ns_list; 198 199 TAILQ_ENTRY(spdk_nvmf_ctrlr) link; 200 }; 201 202 struct spdk_nvmf_subsystem { 203 struct spdk_thread *thread; 204 uint32_t id; 205 enum spdk_nvmf_subsystem_state state; 206 207 char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; 208 enum spdk_nvmf_subtype subtype; 209 uint16_t next_cntlid; 210 bool allow_any_host; 211 212 struct spdk_nvmf_tgt *tgt; 213 214 char sn[SPDK_NVME_CTRLR_SN_LEN + 1]; 215 216 /* Array of pointers to namespaces of size max_nsid indexed by nsid - 1 */ 217 struct spdk_nvmf_ns **ns; 218 uint32_t max_nsid; 219 /* This is the maximum allowed nsid to a subsystem */ 220 uint32_t max_allowed_nsid; 221 uint32_t num_allocated_nsid; 222 223 TAILQ_HEAD(, spdk_nvmf_ctrlr) ctrlrs; 224 225 TAILQ_HEAD(, spdk_nvmf_host) hosts; 226 227 TAILQ_HEAD(, spdk_nvmf_listener) listeners; 228 229 TAILQ_ENTRY(spdk_nvmf_subsystem) entries; 230 }; 231 232 struct spdk_nvmf_transport *spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, 233 enum spdk_nvme_transport_type); 234 235 int spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 236 struct spdk_nvmf_transport *transport); 237 int spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 238 struct spdk_nvmf_subsystem *subsystem); 239 int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 240 struct spdk_nvmf_subsystem *subsystem); 241 int spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 242 struct spdk_nvmf_subsystem *subsystem); 243 int spdk_nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 244 struct spdk_nvmf_subsystem *subsystem); 245 int spdk_nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 246 struct spdk_nvmf_subsystem *subsystem); 247 void spdk_nvmf_request_exec(struct spdk_nvmf_request *req); 248 int spdk_nvmf_request_complete(struct spdk_nvmf_request *req); 249 int spdk_nvmf_request_abort(struct spdk_nvmf_request *req); 250 251 void spdk_nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, 252 void *buffer, uint64_t offset, 253 uint32_t length); 254 255 struct spdk_nvmf_qpair *spdk_nvmf_ctrlr_get_qpair(struct spdk_nvmf_ctrlr *ctrlr, uint16_t qid); 256 void spdk_nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr); 257 int spdk_nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req); 258 int spdk_nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req); 259 int spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req); 260 bool spdk_nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr); 261 bool spdk_nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr); 262 void spdk_nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid); 263 264 int spdk_nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata); 265 266 int spdk_nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem, 267 struct spdk_nvmf_ctrlr *ctrlr); 268 void spdk_nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem, 269 struct spdk_nvmf_ctrlr *ctrlr); 270 struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem, 271 uint16_t cntlid); 272 int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr); 273 274 static inline struct spdk_nvmf_ns * 275 _spdk_nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid) 276 { 277 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 278 if (spdk_unlikely(nsid - 1 >= subsystem->max_nsid)) { 279 return NULL; 280 } 281 282 return subsystem->ns[nsid - 1]; 283 } 284 285 static inline bool 286 spdk_nvmf_qpair_is_admin_queue(struct spdk_nvmf_qpair *qpair) 287 { 288 return qpair->qid == 0; 289 } 290 291 #define OBJECT_NVMF_IO 0x30 292 293 #define TRACE_GROUP_NVMF 0x3 294 #define TRACE_NVMF_IO_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x0) 295 #define TRACE_RDMA_READ_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x1) 296 #define TRACE_RDMA_WRITE_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x2) 297 #define TRACE_RDMA_READ_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x3) 298 #define TRACE_RDMA_WRITE_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x4) 299 #define TRACE_NVMF_LIB_READ_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x5) 300 #define TRACE_NVMF_LIB_WRITE_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x6) 301 #define TRACE_NVMF_LIB_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x7) 302 #define TRACE_NVMF_IO_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF, 0x8) 303 304 #endif /* __NVMF_INTERNAL_H__ */ 305