1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /** \file 35 * TCP network implementation abstraction layer 36 */ 37 38 #ifndef SPDK_INTERNAL_SOCK_H 39 #define SPDK_INTERNAL_SOCK_H 40 41 #include "spdk/stdinc.h" 42 #include "spdk/sock.h" 43 #include "spdk/queue.h" 44 #include "spdk/likely.h" 45 46 #ifdef __cplusplus 47 extern "C" { 48 #endif 49 50 #define MAX_EVENTS_PER_POLL 32 51 #define DEFAULT_SOCK_PRIORITY 0 52 #define MIN_SOCK_PIPE_SIZE 1024 53 #define MIN_SO_RCVBUF_SIZE (2 * 1024 * 1024) 54 #define MIN_SO_SNDBUF_SIZE (2 * 1024 * 1024) 55 #define IOV_BATCH_SIZE 64 56 57 struct spdk_sock { 58 struct spdk_net_impl *net_impl; 59 struct spdk_sock_opts opts; 60 struct spdk_sock_group_impl *group_impl; 61 TAILQ_ENTRY(spdk_sock) link; 62 63 TAILQ_HEAD(, spdk_sock_request) queued_reqs; 64 TAILQ_HEAD(, spdk_sock_request) pending_reqs; 65 int queued_iovcnt; 66 int cb_cnt; 67 spdk_sock_cb cb_fn; 68 void *cb_arg; 69 struct { 70 uint8_t closed : 1; 71 uint8_t reserved : 7; 72 } flags; 73 }; 74 75 struct spdk_sock_group { 76 STAILQ_HEAD(, spdk_sock_group_impl) group_impls; 77 void *ctx; 78 }; 79 80 struct spdk_sock_group_impl { 81 struct spdk_net_impl *net_impl; 82 struct spdk_sock_group *group; 83 TAILQ_HEAD(, spdk_sock) socks; 84 STAILQ_ENTRY(spdk_sock_group_impl) link; 85 }; 86 87 struct spdk_sock_map { 88 STAILQ_HEAD(, spdk_sock_placement_id_entry) entries; 89 pthread_mutex_t mtx; 90 }; 91 92 struct spdk_net_impl { 93 const char *name; 94 int priority; 95 96 int (*getaddr)(struct spdk_sock *sock, char *saddr, int slen, uint16_t *sport, char *caddr, 97 int clen, uint16_t *cport); 98 struct spdk_sock *(*connect)(const char *ip, int port, struct spdk_sock_opts *opts); 99 struct spdk_sock *(*listen)(const char *ip, int port, struct spdk_sock_opts *opts); 100 struct spdk_sock *(*accept)(struct spdk_sock *sock); 101 int (*close)(struct spdk_sock *sock); 102 ssize_t (*recv)(struct spdk_sock *sock, void *buf, size_t len); 103 ssize_t (*readv)(struct spdk_sock *sock, struct iovec *iov, int iovcnt); 104 ssize_t (*writev)(struct spdk_sock *sock, struct iovec *iov, int iovcnt); 105 106 void (*writev_async)(struct spdk_sock *sock, struct spdk_sock_request *req); 107 int (*flush)(struct spdk_sock *sock); 108 109 int (*set_recvlowat)(struct spdk_sock *sock, int nbytes); 110 int (*set_recvbuf)(struct spdk_sock *sock, int sz); 111 int (*set_sendbuf)(struct spdk_sock *sock, int sz); 112 113 bool (*is_ipv6)(struct spdk_sock *sock); 114 bool (*is_ipv4)(struct spdk_sock *sock); 115 bool (*is_connected)(struct spdk_sock *sock); 116 117 struct spdk_sock_group_impl *(*group_impl_get_optimal)(struct spdk_sock *sock); 118 struct spdk_sock_group_impl *(*group_impl_create)(void); 119 int (*group_impl_add_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock); 120 int (*group_impl_remove_sock)(struct spdk_sock_group_impl *group, struct spdk_sock *sock); 121 int (*group_impl_poll)(struct spdk_sock_group_impl *group, int max_events, 122 struct spdk_sock **socks); 123 int (*group_impl_close)(struct spdk_sock_group_impl *group); 124 125 int (*get_opts)(struct spdk_sock_impl_opts *opts, size_t *len); 126 int (*set_opts)(const struct spdk_sock_impl_opts *opts, size_t len); 127 128 STAILQ_ENTRY(spdk_net_impl) link; 129 }; 130 131 void spdk_net_impl_register(struct spdk_net_impl *impl, int priority); 132 133 #define SPDK_NET_IMPL_REGISTER(name, impl, priority) \ 134 static void __attribute__((constructor)) net_impl_register_##name(void) \ 135 { \ 136 spdk_net_impl_register(impl, priority); \ 137 } 138 139 static inline void 140 spdk_sock_request_queue(struct spdk_sock *sock, struct spdk_sock_request *req) 141 { 142 TAILQ_INSERT_TAIL(&sock->queued_reqs, req, internal.link); 143 sock->queued_iovcnt += req->iovcnt; 144 } 145 146 static inline void 147 spdk_sock_request_pend(struct spdk_sock *sock, struct spdk_sock_request *req) 148 { 149 TAILQ_REMOVE(&sock->queued_reqs, req, internal.link); 150 assert(sock->queued_iovcnt >= req->iovcnt); 151 sock->queued_iovcnt -= req->iovcnt; 152 TAILQ_INSERT_TAIL(&sock->pending_reqs, req, internal.link); 153 } 154 155 static inline int 156 spdk_sock_request_put(struct spdk_sock *sock, struct spdk_sock_request *req, int err) 157 { 158 bool closed; 159 int rc = 0; 160 161 TAILQ_REMOVE(&sock->pending_reqs, req, internal.link); 162 163 req->internal.offset = 0; 164 165 closed = sock->flags.closed; 166 sock->cb_cnt++; 167 req->cb_fn(req->cb_arg, err); 168 assert(sock->cb_cnt > 0); 169 sock->cb_cnt--; 170 171 if (sock->cb_cnt == 0 && !closed && sock->flags.closed) { 172 /* The user closed the socket in response to a callback above. */ 173 rc = -1; 174 spdk_sock_close(&sock); 175 } 176 177 return rc; 178 } 179 180 static inline int 181 spdk_sock_abort_requests(struct spdk_sock *sock) 182 { 183 struct spdk_sock_request *req; 184 bool closed; 185 int rc = 0; 186 187 closed = sock->flags.closed; 188 sock->cb_cnt++; 189 190 req = TAILQ_FIRST(&sock->pending_reqs); 191 while (req) { 192 TAILQ_REMOVE(&sock->pending_reqs, req, internal.link); 193 194 req->cb_fn(req->cb_arg, -ECANCELED); 195 196 req = TAILQ_FIRST(&sock->pending_reqs); 197 } 198 199 req = TAILQ_FIRST(&sock->queued_reqs); 200 while (req) { 201 TAILQ_REMOVE(&sock->queued_reqs, req, internal.link); 202 203 assert(sock->queued_iovcnt >= req->iovcnt); 204 sock->queued_iovcnt -= req->iovcnt; 205 206 req->cb_fn(req->cb_arg, -ECANCELED); 207 208 req = TAILQ_FIRST(&sock->queued_reqs); 209 } 210 assert(sock->cb_cnt > 0); 211 sock->cb_cnt--; 212 213 assert(TAILQ_EMPTY(&sock->queued_reqs)); 214 assert(TAILQ_EMPTY(&sock->pending_reqs)); 215 216 if (sock->cb_cnt == 0 && !closed && sock->flags.closed) { 217 /* The user closed the socket in response to a callback above. */ 218 rc = -1; 219 spdk_sock_close(&sock); 220 } 221 222 return rc; 223 } 224 225 static inline int 226 spdk_sock_prep_reqs(struct spdk_sock *_sock, struct iovec *iovs, int index, 227 struct spdk_sock_request **last_req) 228 { 229 int iovcnt, i; 230 struct spdk_sock_request *req; 231 unsigned int offset; 232 233 /* Gather an iov */ 234 iovcnt = index; 235 if (spdk_unlikely(iovcnt >= IOV_BATCH_SIZE)) { 236 goto end; 237 } 238 239 if (last_req != NULL && *last_req != NULL) { 240 req = TAILQ_NEXT(*last_req, internal.link); 241 } else { 242 req = TAILQ_FIRST(&_sock->queued_reqs); 243 } 244 245 while (req) { 246 offset = req->internal.offset; 247 248 for (i = 0; i < req->iovcnt; i++) { 249 /* Consume any offset first */ 250 if (offset >= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len) { 251 offset -= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len; 252 continue; 253 } 254 255 iovs[iovcnt].iov_base = SPDK_SOCK_REQUEST_IOV(req, i)->iov_base + offset; 256 iovs[iovcnt].iov_len = SPDK_SOCK_REQUEST_IOV(req, i)->iov_len - offset; 257 iovcnt++; 258 259 offset = 0; 260 261 if (iovcnt >= IOV_BATCH_SIZE) { 262 break; 263 } 264 } 265 if (iovcnt >= IOV_BATCH_SIZE) { 266 break; 267 } 268 269 if (last_req != NULL) { 270 *last_req = req; 271 } 272 req = TAILQ_NEXT(req, internal.link); 273 } 274 275 end: 276 return iovcnt; 277 } 278 279 static inline void 280 spdk_sock_get_placement_id(int fd, enum spdk_placement_mode mode, int *placement_id) 281 { 282 *placement_id = -1; 283 284 switch (mode) { 285 case PLACEMENT_NONE: 286 break; 287 case PLACEMENT_MARK: 288 case PLACEMENT_NAPI: { 289 #if defined(SO_INCOMING_NAPI_ID) 290 socklen_t len = sizeof(int); 291 292 getsockopt(fd, SOL_SOCKET, SO_INCOMING_NAPI_ID, placement_id, &len); 293 #endif 294 break; 295 } 296 case PLACEMENT_CPU: { 297 #if defined(SO_INCOMING_CPU) 298 socklen_t len = sizeof(int); 299 300 getsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, placement_id, &len); 301 #endif 302 break; 303 } 304 default: 305 break; 306 } 307 } 308 309 /** 310 * Insert a group into the placement map. 311 * If the group is already in the map, take a reference. 312 */ 313 int spdk_sock_map_insert(struct spdk_sock_map *map, int placement_id, 314 struct spdk_sock_group_impl *group_impl); 315 316 /** 317 * Release a reference for the given placement_id. If the reference count goes to 0, the 318 * entry will no longer be associated with a group. 319 */ 320 void spdk_sock_map_release(struct spdk_sock_map *map, int placement_id); 321 322 /** 323 * Look up the group for the given placement_id. 324 */ 325 int spdk_sock_map_lookup(struct spdk_sock_map *map, int placement_id, 326 struct spdk_sock_group_impl **group_impl); 327 328 /** 329 * Find a placement id with no associated group 330 */ 331 int spdk_sock_map_find_free(struct spdk_sock_map *map); 332 333 /** 334 * Clean up all memory associated with the given map 335 */ 336 void spdk_sock_map_cleanup(struct spdk_sock_map *map); 337 338 #ifdef __cplusplus 339 } 340 #endif 341 342 #endif /* SPDK_INTERNAL_SOCK_H */ 343