1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved. 5 */ 6 7 #include "nvme_internal.h" 8 9 struct spdk_nvme_poll_group * 10 spdk_nvme_poll_group_create(void *ctx, struct spdk_nvme_accel_fn_table *table) 11 { 12 struct spdk_nvme_poll_group *group; 13 14 group = calloc(1, sizeof(*group)); 15 if (group == NULL) { 16 return NULL; 17 } 18 19 group->accel_fn_table.table_size = sizeof(struct spdk_nvme_accel_fn_table); 20 if (table && table->table_size != 0) { 21 group->accel_fn_table.table_size = table->table_size; 22 #define SET_FIELD(field) \ 23 if (offsetof(struct spdk_nvme_accel_fn_table, field) + sizeof(table->field) <= table->table_size) { \ 24 group->accel_fn_table.field = table->field; \ 25 } \ 26 27 SET_FIELD(submit_accel_crc32c); 28 /* Do not remove this statement, you should always update this statement when you adding a new field, 29 * and do not forget to add the SET_FIELD statement for your added field. */ 30 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_accel_fn_table) == 16, "Incorrect size"); 31 32 #undef SET_FIELD 33 } 34 35 group->ctx = ctx; 36 STAILQ_INIT(&group->tgroups); 37 38 return group; 39 } 40 41 struct spdk_nvme_poll_group * 42 spdk_nvme_qpair_get_optimal_poll_group(struct spdk_nvme_qpair *qpair) 43 { 44 struct spdk_nvme_transport_poll_group *tgroup; 45 46 tgroup = nvme_transport_qpair_get_optimal_poll_group(qpair->transport, qpair); 47 48 if (tgroup == NULL) { 49 return NULL; 50 } 51 52 return tgroup->group; 53 } 54 55 int 56 spdk_nvme_poll_group_add(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 57 { 58 struct spdk_nvme_transport_poll_group *tgroup; 59 const struct spdk_nvme_transport *transport; 60 61 if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) { 62 return -EINVAL; 63 } 64 65 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 66 if (tgroup->transport == qpair->transport) { 67 break; 68 } 69 } 70 71 /* See if a new transport has been added (dlopen style) and we need to update the poll group */ 72 if (!tgroup) { 73 transport = nvme_get_first_transport(); 74 while (transport != NULL) { 75 if (transport == qpair->transport) { 76 tgroup = nvme_transport_poll_group_create(transport); 77 if (tgroup == NULL) { 78 return -ENOMEM; 79 } 80 tgroup->group = group; 81 STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 82 break; 83 } 84 transport = nvme_get_next_transport(transport); 85 } 86 } 87 88 return tgroup ? nvme_transport_poll_group_add(tgroup, qpair) : -ENODEV; 89 } 90 91 int 92 spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme_qpair *qpair) 93 { 94 struct spdk_nvme_transport_poll_group *tgroup; 95 96 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 97 if (tgroup->transport == qpair->transport) { 98 return nvme_transport_poll_group_remove(tgroup, qpair); 99 } 100 } 101 102 return -ENODEV; 103 } 104 105 int 106 nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair) 107 { 108 return nvme_transport_poll_group_connect_qpair(qpair); 109 } 110 111 int 112 nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair) 113 { 114 return nvme_transport_poll_group_disconnect_qpair(qpair); 115 } 116 117 int64_t 118 spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group, 119 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb) 120 { 121 struct spdk_nvme_transport_poll_group *tgroup; 122 int64_t local_completions = 0, error_reason = 0, num_completions = 0; 123 124 if (disconnected_qpair_cb == NULL) { 125 return -EINVAL; 126 } 127 128 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 129 local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair, 130 disconnected_qpair_cb); 131 if (local_completions < 0 && error_reason == 0) { 132 error_reason = local_completions; 133 } else { 134 num_completions += local_completions; 135 /* Just to be safe */ 136 assert(num_completions >= 0); 137 } 138 } 139 140 return error_reason ? error_reason : num_completions; 141 } 142 143 int 144 spdk_nvme_poll_group_all_connected(struct spdk_nvme_poll_group *group) 145 { 146 struct spdk_nvme_transport_poll_group *tgroup; 147 struct spdk_nvme_qpair *qpair; 148 int rc = 0; 149 150 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 151 if (!STAILQ_EMPTY(&tgroup->disconnected_qpairs)) { 152 /* Treat disconnected qpairs as highest priority for notification. 153 * This means we can just return immediately here. 154 */ 155 return -EIO; 156 } 157 STAILQ_FOREACH(qpair, &tgroup->connected_qpairs, poll_group_stailq) { 158 if (nvme_qpair_get_state(qpair) < NVME_QPAIR_CONNECTING) { 159 return -EIO; 160 } else if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) { 161 rc = -EAGAIN; 162 /* Break so that we can check the remaining transport groups, 163 * in case any of them have a disconnected qpair. 164 */ 165 break; 166 } 167 } 168 } 169 170 return rc; 171 } 172 173 void * 174 spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group) 175 { 176 return group->ctx; 177 } 178 179 int 180 spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group) 181 { 182 struct spdk_nvme_transport_poll_group *tgroup, *tmp_tgroup; 183 184 STAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp_tgroup) { 185 STAILQ_REMOVE(&group->tgroups, tgroup, spdk_nvme_transport_poll_group, link); 186 if (nvme_transport_poll_group_destroy(tgroup) != 0) { 187 STAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 188 return -EBUSY; 189 } 190 191 } 192 193 free(group); 194 195 return 0; 196 } 197 198 int 199 spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group, 200 struct spdk_nvme_poll_group_stat **stats) 201 { 202 struct spdk_nvme_transport_poll_group *tgroup; 203 struct spdk_nvme_poll_group_stat *result; 204 uint32_t transports_count = 0; 205 /* Not all transports used by this poll group may support statistics reporting */ 206 uint32_t reported_stats_count = 0; 207 int rc; 208 209 assert(group); 210 assert(stats); 211 212 result = calloc(1, sizeof(*result)); 213 if (!result) { 214 SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n"); 215 return -ENOMEM; 216 } 217 218 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 219 transports_count++; 220 } 221 222 result->transport_stat = calloc(transports_count, sizeof(*result->transport_stat)); 223 if (!result->transport_stat) { 224 SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n"); 225 free(result); 226 return -ENOMEM; 227 } 228 229 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 230 rc = nvme_transport_poll_group_get_stats(tgroup, &result->transport_stat[reported_stats_count]); 231 if (rc == 0) { 232 reported_stats_count++; 233 } 234 } 235 236 if (reported_stats_count == 0) { 237 free(result->transport_stat); 238 free(result); 239 SPDK_DEBUGLOG(nvme, "No transport statistics available\n"); 240 return -ENOTSUP; 241 } 242 243 result->num_transports = reported_stats_count; 244 *stats = result; 245 246 return 0; 247 } 248 249 void 250 spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group, 251 struct spdk_nvme_poll_group_stat *stat) 252 { 253 struct spdk_nvme_transport_poll_group *tgroup; 254 uint32_t i; 255 uint32_t freed_stats __attribute__((unused)) = 0; 256 257 assert(group); 258 assert(stat); 259 260 for (i = 0; i < stat->num_transports; i++) { 261 STAILQ_FOREACH(tgroup, &group->tgroups, link) { 262 if (nvme_transport_get_trtype(tgroup->transport) == stat->transport_stat[i]->trtype) { 263 nvme_transport_poll_group_free_stats(tgroup, stat->transport_stat[i]); 264 freed_stats++; 265 break; 266 } 267 } 268 } 269 270 assert(freed_stats == stat->num_transports); 271 272 free(stat->transport_stat); 273 free(stat); 274 } 275