1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved.
3 */
4
5 #include <stdlib.h>
6 #include <fcntl.h>
7 #include <unistd.h>
8 #include <sys/types.h>
9 #include <sys/socket.h>
10 #include <sys/ioctl.h>
11 #include <errno.h>
12
13 #include <rte_version.h>
14 #include <rte_mbuf.h>
15 #include <rte_ether.h>
16 #include <ethdev_driver.h>
17 #include <ethdev_vdev.h>
18 #include <rte_malloc.h>
19 #include <rte_kvargs.h>
20 #include <bus_vdev_driver.h>
21 #include <rte_hash.h>
22 #include <rte_jhash.h>
23 #include <rte_string_fns.h>
24
25 #include "rte_eth_memif.h"
26 #include "memif_socket.h"
27
28 static void memif_intr_handler(void *arg);
29
30 static ssize_t
memif_msg_send(int fd,memif_msg_t * msg,int afd)31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
32 {
33 struct msghdr mh = { 0 };
34 struct iovec iov[1];
35 struct cmsghdr *cmsg;
36 char ctl[CMSG_SPACE(sizeof(int))];
37
38 iov[0].iov_base = msg;
39 iov[0].iov_len = sizeof(memif_msg_t);
40 mh.msg_iov = iov;
41 mh.msg_iovlen = 1;
42
43 if (afd > 0) {
44 memset(&ctl, 0, sizeof(ctl));
45 mh.msg_control = ctl;
46 mh.msg_controllen = sizeof(ctl);
47 cmsg = CMSG_FIRSTHDR(&mh);
48 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
49 cmsg->cmsg_level = SOL_SOCKET;
50 cmsg->cmsg_type = SCM_RIGHTS;
51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
52 }
53
54 return sendmsg(fd, &mh, 0);
55 }
56
57 static int
memif_msg_send_from_queue(struct memif_control_channel * cc)58 memif_msg_send_from_queue(struct memif_control_channel *cc)
59 {
60 ssize_t size;
61 int ret = 0;
62 struct memif_msg_queue_elt *e;
63
64 e = TAILQ_FIRST(&cc->msg_queue);
65 if (e == NULL)
66 return 0;
67
68 if (rte_intr_fd_get(cc->intr_handle) < 0)
69 return -1;
70
71 size = memif_msg_send(rte_intr_fd_get(cc->intr_handle), &e->msg,
72 e->fd);
73 if (size != sizeof(memif_msg_t)) {
74 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
75 ret = -1;
76 } else {
77 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
78 }
79 TAILQ_REMOVE(&cc->msg_queue, e, next);
80 rte_free(e);
81
82 return ret;
83 }
84
85 static struct memif_msg_queue_elt *
memif_msg_enq(struct memif_control_channel * cc)86 memif_msg_enq(struct memif_control_channel *cc)
87 {
88 struct memif_msg_queue_elt *e;
89
90 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
91 if (e == NULL) {
92 MIF_LOG(ERR, "Failed to allocate control message.");
93 return NULL;
94 }
95
96 e->fd = -1;
97 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
98
99 return e;
100 }
101
102 void
memif_msg_enq_disconnect(struct memif_control_channel * cc,const char * reason,int err_code)103 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
104 int err_code)
105 {
106 struct memif_msg_queue_elt *e;
107 struct pmd_internals *pmd;
108 memif_msg_disconnect_t *d;
109
110 if (cc == NULL) {
111 MIF_LOG(DEBUG, "Missing control channel.");
112 return;
113 }
114
115 e = memif_msg_enq(cc);
116 if (e == NULL) {
117 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
118 return;
119 }
120
121 d = &e->msg.disconnect;
122
123 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
124 d->code = err_code;
125
126 if (reason != NULL) {
127 strlcpy((char *)d->string, reason, sizeof(d->string));
128 if (cc->dev != NULL) {
129 pmd = cc->dev->data->dev_private;
130 strlcpy(pmd->local_disc_string, reason,
131 sizeof(pmd->local_disc_string));
132 }
133 }
134 }
135
136 static int
memif_msg_enq_hello(struct memif_control_channel * cc)137 memif_msg_enq_hello(struct memif_control_channel *cc)
138 {
139 struct memif_msg_queue_elt *e = memif_msg_enq(cc);
140 memif_msg_hello_t *h;
141
142 if (e == NULL)
143 return -1;
144
145 h = &e->msg.hello;
146
147 e->msg.type = MEMIF_MSG_TYPE_HELLO;
148 h->min_version = MEMIF_VERSION;
149 h->max_version = MEMIF_VERSION;
150 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
151 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
152 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
153 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
154
155 strlcpy((char *)h->name, rte_version(), sizeof(h->name));
156
157 return 0;
158 }
159
160 static int
memif_msg_receive_hello(struct rte_eth_dev * dev,memif_msg_t * msg)161 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
162 {
163 struct pmd_internals *pmd = dev->data->dev_private;
164 memif_msg_hello_t *h = &msg->hello;
165
166 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
167 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
168 return -1;
169 }
170
171 /* Set parameters for active connection */
172 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1,
173 pmd->cfg.num_c2s_rings);
174 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1,
175 pmd->cfg.num_s2c_rings);
176 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
177 pmd->cfg.log2_ring_size);
178 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
179
180 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
181
182 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
183
184 return 0;
185 }
186
187 static int
memif_msg_receive_init(struct memif_control_channel * cc,memif_msg_t * msg)188 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
189 {
190 memif_msg_init_t *i = &msg->init;
191 struct memif_socket_dev_list_elt *elt;
192 struct pmd_internals *pmd;
193 struct rte_eth_dev *dev;
194
195 if (i->version != MEMIF_VERSION) {
196 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
197 return -1;
198 }
199
200 if (cc->socket == NULL) {
201 memif_msg_enq_disconnect(cc, "Device error", 0);
202 return -1;
203 }
204
205 /* Find device with requested ID */
206 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
207 dev = elt->dev;
208 pmd = dev->data->dev_private;
209 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
210 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) {
211 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
212 ETH_MEMIF_FLAG_CONNECTED)) {
213 memif_msg_enq_disconnect(cc,
214 "Already connected", 0);
215 return -1;
216 }
217
218 /* assign control channel to device */
219 cc->dev = dev;
220 pmd->cc = cc;
221
222 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
223 memif_msg_enq_disconnect(pmd->cc,
224 "Only ethernet mode supported",
225 0);
226 return -1;
227 }
228
229 strlcpy(pmd->remote_name, (char *)i->name,
230 sizeof(pmd->remote_name));
231
232 if (*pmd->secret != '\0') {
233 if (*i->secret == '\0') {
234 memif_msg_enq_disconnect(pmd->cc,
235 "Secret required", 0);
236 return -1;
237 }
238 if (strncmp(pmd->secret, (char *)i->secret,
239 ETH_MEMIF_SECRET_SIZE) != 0) {
240 memif_msg_enq_disconnect(pmd->cc,
241 "Incorrect secret", 0);
242 return -1;
243 }
244 }
245
246 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
247 return 0;
248 }
249 }
250
251 /* ID not found on this socket */
252 MIF_LOG(DEBUG, "ID %u not found.", i->id);
253 memif_msg_enq_disconnect(cc, "ID not found", 0);
254 return -1;
255 }
256
257 static int
memif_msg_receive_add_region(struct rte_eth_dev * dev,memif_msg_t * msg,int fd)258 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
259 int fd)
260 {
261 struct pmd_internals *pmd = dev->data->dev_private;
262 struct pmd_process_private *proc_private = dev->process_private;
263 memif_msg_add_region_t *ar = &msg->add_region;
264 struct memif_region *r;
265
266 if (fd < 0) {
267 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
268 return -1;
269 }
270
271 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
272 ar->index != proc_private->regions_num ||
273 proc_private->regions[ar->index] != NULL) {
274 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
275 return -1;
276 }
277
278 r = rte_zmalloc("region", sizeof(struct memif_region), 0);
279 if (r == NULL) {
280 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
281 return -ENOMEM;
282 }
283
284 r->fd = fd;
285 r->region_size = ar->size;
286 r->addr = NULL;
287
288 proc_private->regions[ar->index] = r;
289 proc_private->regions_num++;
290
291 return 0;
292 }
293
294 static int
memif_msg_receive_add_ring(struct rte_eth_dev * dev,memif_msg_t * msg,int fd)295 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
296 {
297 struct pmd_internals *pmd = dev->data->dev_private;
298 memif_msg_add_ring_t *ar = &msg->add_ring;
299 struct memif_queue *mq;
300
301 if (fd < 0) {
302 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
303 return -1;
304 }
305
306 /* check if we have enough queues */
307 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) {
308 if (ar->index >= pmd->cfg.num_c2s_rings) {
309 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
310 return -1;
311 }
312 pmd->run.num_c2s_rings++;
313 } else {
314 if (ar->index >= pmd->cfg.num_s2c_rings) {
315 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
316 return -1;
317 }
318 pmd->run.num_s2c_rings++;
319 }
320
321 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ?
322 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
323
324 if (rte_intr_fd_set(mq->intr_handle, fd))
325 return -1;
326
327 mq->log2_ring_size = ar->log2_ring_size;
328 mq->region = ar->region;
329 mq->ring_offset = ar->offset;
330
331 return 0;
332 }
333
334 static int
memif_msg_receive_connect(struct rte_eth_dev * dev,memif_msg_t * msg)335 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
336 {
337 struct pmd_internals *pmd = dev->data->dev_private;
338 memif_msg_connect_t *c = &msg->connect;
339 int ret;
340
341 ret = memif_connect(dev);
342 if (ret < 0)
343 return ret;
344
345 strlcpy(pmd->remote_if_name, (char *)c->if_name,
346 sizeof(pmd->remote_if_name));
347 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
348
349 return 0;
350 }
351
352 static int
memif_msg_receive_connected(struct rte_eth_dev * dev,memif_msg_t * msg)353 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
354 {
355 struct pmd_internals *pmd = dev->data->dev_private;
356 memif_msg_connected_t *c = &msg->connected;
357 int ret;
358
359 ret = memif_connect(dev);
360 if (ret < 0)
361 return ret;
362
363 strlcpy(pmd->remote_if_name, (char *)c->if_name,
364 sizeof(pmd->remote_if_name));
365 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
366
367 return 0;
368 }
369
370 static int
memif_msg_receive_disconnect(struct rte_eth_dev * dev,memif_msg_t * msg)371 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
372 {
373 struct pmd_internals *pmd = dev->data->dev_private;
374 memif_msg_disconnect_t *d = &msg->disconnect;
375
376 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
377 strlcpy(pmd->remote_disc_string, (char *)d->string,
378 sizeof(pmd->remote_disc_string));
379
380 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
381
382 memset(pmd->local_disc_string, 0, 96);
383 memif_disconnect(dev);
384 return 0;
385 }
386
387 static int
memif_msg_enq_ack(struct rte_eth_dev * dev)388 memif_msg_enq_ack(struct rte_eth_dev *dev)
389 {
390 struct pmd_internals *pmd = dev->data->dev_private;
391 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
392 if (e == NULL)
393 return -1;
394
395 e->msg.type = MEMIF_MSG_TYPE_ACK;
396
397 return 0;
398 }
399
400 static int
memif_msg_enq_init(struct rte_eth_dev * dev)401 memif_msg_enq_init(struct rte_eth_dev *dev)
402 {
403 struct pmd_internals *pmd = dev->data->dev_private;
404 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
405 memif_msg_init_t *i;
406
407 if (e == NULL)
408 return -1;
409 i = &e->msg.init;
410 e->msg.type = MEMIF_MSG_TYPE_INIT;
411 i->version = MEMIF_VERSION;
412 i->id = pmd->id;
413 i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
414
415 strlcpy((char *)i->name, rte_version(), sizeof(i->name));
416
417 if (*pmd->secret != '\0')
418 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
419
420 return 0;
421 }
422
423 static int
memif_msg_enq_add_region(struct rte_eth_dev * dev,uint8_t idx)424 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
425 {
426 struct pmd_internals *pmd = dev->data->dev_private;
427 struct pmd_process_private *proc_private = dev->process_private;
428 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
429 memif_msg_add_region_t *ar;
430 struct memif_region *mr = proc_private->regions[idx];
431
432 if (e == NULL)
433 return -1;
434
435 ar = &e->msg.add_region;
436 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
437 e->fd = mr->fd;
438 ar->index = idx;
439 ar->size = mr->region_size;
440
441 return 0;
442 }
443
444 static int
memif_msg_enq_add_ring(struct rte_eth_dev * dev,uint8_t idx,memif_ring_type_t type)445 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
446 memif_ring_type_t type)
447 {
448 struct pmd_internals *pmd = dev->data->dev_private;
449 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
450 struct memif_queue *mq;
451 memif_msg_add_ring_t *ar;
452
453 if (e == NULL)
454 return -1;
455
456 ar = &e->msg.add_ring;
457 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] :
458 dev->data->rx_queues[idx];
459
460 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
461 e->fd = rte_intr_fd_get(mq->intr_handle);
462 ar->index = idx;
463 ar->offset = mq->ring_offset;
464 ar->region = mq->region;
465 ar->log2_ring_size = mq->log2_ring_size;
466 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0;
467 ar->private_hdr_size = 0;
468
469 return 0;
470 }
471
472 static int
memif_msg_enq_connect(struct rte_eth_dev * dev)473 memif_msg_enq_connect(struct rte_eth_dev *dev)
474 {
475 struct pmd_internals *pmd = dev->data->dev_private;
476 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
477 memif_msg_connect_t *c;
478
479 if (e == NULL)
480 return -1;
481
482 c = &e->msg.connect;
483 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
484 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
485
486 return 0;
487 }
488
489 static int
memif_msg_enq_connected(struct rte_eth_dev * dev)490 memif_msg_enq_connected(struct rte_eth_dev *dev)
491 {
492 struct pmd_internals *pmd = dev->data->dev_private;
493 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
494 memif_msg_connected_t *c;
495
496 if (e == NULL)
497 return -1;
498
499 c = &e->msg.connected;
500 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
501 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
502
503 return 0;
504 }
505
506 static void
memif_intr_unregister_handler(struct rte_intr_handle * intr_handle,void * arg)507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
508 {
509 struct memif_msg_queue_elt *elt;
510 struct memif_control_channel *cc = arg;
511
512 /* close control channel fd */
513 if (rte_intr_fd_get(intr_handle) >= 0)
514 close(rte_intr_fd_get(intr_handle));
515 /* clear message queue */
516 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
517 TAILQ_REMOVE(&cc->msg_queue, elt, next);
518 rte_free(elt);
519 }
520 rte_intr_instance_free(cc->intr_handle);
521 /* free control channel */
522 rte_free(cc);
523 }
524
525 void
memif_disconnect(struct rte_eth_dev * dev)526 memif_disconnect(struct rte_eth_dev *dev)
527 {
528 struct pmd_internals *pmd = dev->data->dev_private;
529 struct memif_msg_queue_elt *elt, *next;
530 struct memif_queue *mq;
531 struct rte_intr_handle *ih;
532 int i;
533 int ret;
534
535 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
536 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
537 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
538
539 rte_spinlock_lock(&pmd->cc_lock);
540 if (pmd->cc != NULL) {
541 /* Clear control message queue (except disconnect message if any). */
542 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
543 next = TAILQ_NEXT(elt, next);
544 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
545 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
546 rte_free(elt);
547 }
548 }
549 /* send disconnect message (if there is any in queue) */
550 memif_msg_send_from_queue(pmd->cc);
551
552 /* at this point, there should be no more messages in queue */
553 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
554 MIF_LOG(WARNING,
555 "Unexpected message(s) in message queue.");
556 }
557
558 ih = pmd->cc->intr_handle;
559 if (rte_intr_fd_get(ih) > 0) {
560 ret = rte_intr_callback_unregister(ih,
561 memif_intr_handler,
562 pmd->cc);
563 /*
564 * If callback is active (disconnecting based on
565 * received control message).
566 */
567 if (ret == -EAGAIN) {
568 ret = rte_intr_callback_unregister_pending(ih,
569 memif_intr_handler,
570 pmd->cc,
571 memif_intr_unregister_handler);
572 } else if (ret > 0) {
573 close(rte_intr_fd_get(ih));
574 rte_intr_instance_free(ih);
575 rte_free(pmd->cc);
576 }
577 pmd->cc = NULL;
578 if (ret <= 0)
579 MIF_LOG(WARNING,
580 "Failed to unregister control channel callback.");
581 }
582 }
583 rte_spinlock_unlock(&pmd->cc_lock);
584
585 /* unconfig interrupts */
586 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) {
587 if (pmd->role == MEMIF_ROLE_CLIENT) {
588 if (dev->data->tx_queues != NULL)
589 mq = dev->data->tx_queues[i];
590 else
591 continue;
592 } else {
593 if (dev->data->rx_queues != NULL)
594 mq = dev->data->rx_queues[i];
595 else
596 continue;
597 }
598
599 if (rte_intr_fd_get(mq->intr_handle) > 0) {
600 close(rte_intr_fd_get(mq->intr_handle));
601 rte_intr_fd_set(mq->intr_handle, -1);
602 }
603 }
604 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) {
605 if (pmd->role == MEMIF_ROLE_SERVER) {
606 if (dev->data->tx_queues != NULL)
607 mq = dev->data->tx_queues[i];
608 else
609 continue;
610 } else {
611 if (dev->data->rx_queues != NULL)
612 mq = dev->data->rx_queues[i];
613 else
614 continue;
615 }
616
617 if (rte_intr_fd_get(mq->intr_handle) > 0) {
618 close(rte_intr_fd_get(mq->intr_handle));
619 rte_intr_fd_set(mq->intr_handle, -1);
620 }
621 }
622
623 memif_free_regions(dev);
624
625 /* reset connection configuration */
626 memset(&pmd->run, 0, sizeof(pmd->run));
627
628 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
629 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client");
630 }
631
632 static int
memif_msg_receive(struct memif_control_channel * cc)633 memif_msg_receive(struct memif_control_channel *cc)
634 {
635 char ctl[CMSG_SPACE(sizeof(int)) +
636 CMSG_SPACE(sizeof(struct ucred))] = { 0 };
637 struct msghdr mh = { 0 };
638 struct iovec iov[1];
639 memif_msg_t msg = { 0 };
640 ssize_t size;
641 int ret = 0;
642 struct ucred *cr __rte_unused;
643 cr = 0;
644 struct cmsghdr *cmsg;
645 int afd = -1;
646 int i;
647 struct pmd_internals *pmd;
648 struct pmd_process_private *proc_private;
649
650 iov[0].iov_base = (void *)&msg;
651 iov[0].iov_len = sizeof(memif_msg_t);
652 mh.msg_iov = iov;
653 mh.msg_iovlen = 1;
654 mh.msg_control = ctl;
655 mh.msg_controllen = sizeof(ctl);
656
657 if (rte_intr_fd_get(cc->intr_handle) < 0)
658 return -1;
659
660 size = recvmsg(rte_intr_fd_get(cc->intr_handle), &mh, 0);
661 if (size != sizeof(memif_msg_t)) {
662 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
663 if (size > 0)
664 /* 0 means end-of-file, negative size means error,
665 * don't send further disconnect message in such cases.
666 */
667 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
668 return -1;
669 }
670 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
671
672 cmsg = CMSG_FIRSTHDR(&mh);
673 while (cmsg) {
674 if (cmsg->cmsg_level == SOL_SOCKET) {
675 if (cmsg->cmsg_type == SCM_CREDENTIALS)
676 cr = (struct ucred *)CMSG_DATA(cmsg);
677 else if (cmsg->cmsg_type == SCM_RIGHTS)
678 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
679 }
680 cmsg = CMSG_NXTHDR(&mh, cmsg);
681 }
682
683 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
684 MIF_LOG(DEBUG, "Unexpected message.");
685 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
686 return -1;
687 }
688
689 /* get device from hash data */
690 switch (msg.type) {
691 case MEMIF_MSG_TYPE_ACK:
692 break;
693 case MEMIF_MSG_TYPE_HELLO:
694 ret = memif_msg_receive_hello(cc->dev, &msg);
695 if (ret < 0)
696 goto exit;
697 ret = memif_init_regions_and_queues(cc->dev);
698 if (ret < 0)
699 goto exit;
700 ret = memif_msg_enq_init(cc->dev);
701 if (ret < 0)
702 goto exit;
703 pmd = cc->dev->data->dev_private;
704 proc_private = cc->dev->process_private;
705 for (i = 0; i < proc_private->regions_num; i++) {
706 ret = memif_msg_enq_add_region(cc->dev, i);
707 if (ret < 0)
708 goto exit;
709 }
710 for (i = 0; i < pmd->run.num_c2s_rings; i++) {
711 ret = memif_msg_enq_add_ring(cc->dev, i,
712 MEMIF_RING_C2S);
713 if (ret < 0)
714 goto exit;
715 }
716 for (i = 0; i < pmd->run.num_s2c_rings; i++) {
717 ret = memif_msg_enq_add_ring(cc->dev, i,
718 MEMIF_RING_S2C);
719 if (ret < 0)
720 goto exit;
721 }
722 ret = memif_msg_enq_connect(cc->dev);
723 if (ret < 0)
724 goto exit;
725 break;
726 case MEMIF_MSG_TYPE_INIT:
727 /*
728 * This cc does not have an interface associated with it.
729 * If suitable interface is found it will be assigned here.
730 */
731 ret = memif_msg_receive_init(cc, &msg);
732 if (ret < 0)
733 goto exit;
734 ret = memif_msg_enq_ack(cc->dev);
735 if (ret < 0)
736 goto exit;
737 break;
738 case MEMIF_MSG_TYPE_ADD_REGION:
739 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
740 if (ret < 0)
741 goto exit;
742 ret = memif_msg_enq_ack(cc->dev);
743 if (ret < 0)
744 goto exit;
745 break;
746 case MEMIF_MSG_TYPE_ADD_RING:
747 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
748 if (ret < 0)
749 goto exit;
750 ret = memif_msg_enq_ack(cc->dev);
751 if (ret < 0)
752 goto exit;
753 break;
754 case MEMIF_MSG_TYPE_CONNECT:
755 ret = memif_msg_receive_connect(cc->dev, &msg);
756 if (ret < 0)
757 goto exit;
758 ret = memif_msg_enq_connected(cc->dev);
759 if (ret < 0)
760 goto exit;
761 break;
762 case MEMIF_MSG_TYPE_CONNECTED:
763 ret = memif_msg_receive_connected(cc->dev, &msg);
764 break;
765 case MEMIF_MSG_TYPE_DISCONNECT:
766 ret = memif_msg_receive_disconnect(cc->dev, &msg);
767 if (ret < 0)
768 goto exit;
769 break;
770 default:
771 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
772 ret = -1;
773 goto exit;
774 }
775
776 exit:
777 return ret;
778 }
779
780 static void
memif_intr_handler(void * arg)781 memif_intr_handler(void *arg)
782 {
783 struct memif_control_channel *cc = arg;
784 int ret;
785
786 ret = memif_msg_receive(cc);
787 /* if driver failed to assign device */
788 if (cc->dev == NULL) {
789 memif_msg_send_from_queue(cc);
790 ret = rte_intr_callback_unregister_pending(cc->intr_handle,
791 memif_intr_handler,
792 cc,
793 memif_intr_unregister_handler);
794 if (ret < 0)
795 MIF_LOG(WARNING,
796 "Failed to unregister control channel callback.");
797 return;
798 }
799 /* if memif_msg_receive failed */
800 if (ret < 0)
801 goto disconnect;
802
803 ret = memif_msg_send_from_queue(cc);
804 if (ret < 0)
805 goto disconnect;
806
807 return;
808
809 disconnect:
810 if (cc->dev == NULL) {
811 MIF_LOG(WARNING, "eth dev not allocated");
812 return;
813 }
814 memif_disconnect(cc->dev);
815 }
816
817 static void
memif_listener_handler(void * arg)818 memif_listener_handler(void *arg)
819 {
820 struct memif_socket *socket = arg;
821 int sockfd;
822 int addr_len;
823 struct sockaddr_un client;
824 struct memif_control_channel *cc;
825 int ret;
826
827 addr_len = sizeof(client);
828 sockfd = accept(rte_intr_fd_get(socket->intr_handle),
829 (struct sockaddr *)&client, (socklen_t *)&addr_len);
830 if (sockfd < 0) {
831 MIF_LOG(ERR,
832 "Failed to accept connection request on socket fd %d",
833 rte_intr_fd_get(socket->intr_handle));
834 return;
835 }
836
837 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
838
839 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
840 if (cc == NULL) {
841 MIF_LOG(ERR, "Failed to allocate control channel.");
842 goto error;
843 }
844
845 /* Allocate interrupt instance */
846 cc->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
847 if (cc->intr_handle == NULL) {
848 MIF_LOG(ERR, "Failed to allocate intr handle");
849 goto error;
850 }
851
852 if (rte_intr_fd_set(cc->intr_handle, sockfd))
853 goto error;
854
855 if (rte_intr_type_set(cc->intr_handle, RTE_INTR_HANDLE_EXT))
856 goto error;
857
858 cc->socket = socket;
859 cc->dev = NULL;
860 TAILQ_INIT(&cc->msg_queue);
861
862 ret = rte_intr_callback_register(cc->intr_handle, memif_intr_handler,
863 cc);
864 if (ret < 0) {
865 MIF_LOG(ERR, "Failed to register control channel callback.");
866 goto error;
867 }
868
869 ret = memif_msg_enq_hello(cc);
870 if (ret < 0) {
871 MIF_LOG(ERR, "Failed to enqueue hello message.");
872 goto error;
873 }
874 ret = memif_msg_send_from_queue(cc);
875 if (ret < 0)
876 goto error;
877
878 return;
879
880 error:
881 if (sockfd >= 0) {
882 close(sockfd);
883 sockfd = -1;
884 }
885 if (cc != NULL) {
886 rte_intr_instance_free(cc->intr_handle);
887 rte_free(cc);
888 }
889 }
890
891 static struct memif_socket *
memif_socket_create(char * key,uint8_t listener,bool is_abstract,uid_t owner_uid,gid_t owner_gid)892 memif_socket_create(char *key, uint8_t listener, bool is_abstract, uid_t owner_uid, gid_t owner_gid)
893 {
894 struct memif_socket *sock;
895 struct sockaddr_un un = { 0 };
896 uint32_t sunlen;
897 int sockfd;
898 int ret;
899 int on = 1;
900
901 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
902 if (sock == NULL) {
903 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
904 return NULL;
905 }
906
907 sock->listener = listener;
908 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
909 TAILQ_INIT(&sock->dev_queue);
910
911 if (listener != 0) {
912 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
913 if (sockfd < 0)
914 goto error;
915
916 un.sun_family = AF_UNIX;
917 if (is_abstract) {
918 /* abstract address */
919 un.sun_path[0] = '\0';
920 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1);
921 sunlen = RTE_MIN(1 + strlen(sock->filename),
922 MEMIF_SOCKET_UN_SIZE) +
923 sizeof(un) - sizeof(un.sun_path);
924 } else {
925 sunlen = sizeof(un);
926 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
927 }
928
929 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
930 sizeof(on));
931 if (ret < 0)
932 goto error;
933
934 ret = bind(sockfd, (struct sockaddr *)&un, sunlen);
935 if (ret < 0)
936 goto error;
937
938 ret = listen(sockfd, 1);
939 if (ret < 0)
940 goto error;
941
942 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
943
944 if (!is_abstract && (owner_uid != (uid_t)-1 || owner_gid != (gid_t)-1)) {
945 ret = chown(sock->filename, owner_uid, owner_gid);
946 if (ret < 0) {
947 MIF_LOG(ERR, "Failed to change listener socket owner");
948 goto error;
949 }
950 }
951
952 /* Allocate interrupt instance */
953 sock->intr_handle =
954 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
955 if (sock->intr_handle == NULL) {
956 MIF_LOG(ERR, "Failed to allocate intr handle");
957 goto error;
958 }
959
960 if (rte_intr_fd_set(sock->intr_handle, sockfd))
961 goto error;
962
963 if (rte_intr_type_set(sock->intr_handle, RTE_INTR_HANDLE_EXT))
964 goto error;
965
966 ret = rte_intr_callback_register(sock->intr_handle,
967 memif_listener_handler, sock);
968 if (ret < 0) {
969 MIF_LOG(ERR, "Failed to register interrupt "
970 "callback for listener socket");
971 return NULL;
972 }
973 }
974
975 return sock;
976
977 error:
978 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
979 if (sock != NULL) {
980 rte_intr_instance_free(sock->intr_handle);
981 rte_free(sock);
982 }
983 if (sockfd >= 0)
984 close(sockfd);
985 return NULL;
986 }
987
988 static struct rte_hash *
memif_create_socket_hash(void)989 memif_create_socket_hash(void)
990 {
991 struct rte_hash_parameters params = { 0 };
992
993 params.name = MEMIF_SOCKET_HASH_NAME;
994 params.entries = 256;
995 params.key_len = MEMIF_SOCKET_UN_SIZE;
996 params.hash_func = rte_jhash;
997 params.hash_func_init_val = 0;
998 params.socket_id = SOCKET_ID_ANY;
999 return rte_hash_create(¶ms);
1000 }
1001
1002 int
memif_socket_init(struct rte_eth_dev * dev,const char * socket_filename)1003 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
1004 {
1005 struct pmd_internals *pmd = dev->data->dev_private;
1006 struct memif_socket *socket = NULL;
1007 struct memif_socket_dev_list_elt *elt;
1008 struct pmd_internals *tmp_pmd;
1009 struct rte_hash *hash;
1010 int ret;
1011 char key[MEMIF_SOCKET_UN_SIZE];
1012
1013 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1014 if (hash == NULL) {
1015 hash = memif_create_socket_hash();
1016 if (hash == NULL) {
1017 MIF_LOG(ERR, "Failed to create memif socket hash.");
1018 return -1;
1019 }
1020 }
1021
1022 memset(key, 0, MEMIF_SOCKET_UN_SIZE);
1023 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
1024 ret = rte_hash_lookup_data(hash, key, (void **)&socket);
1025 if (ret < 0) {
1026 socket = memif_socket_create(key,
1027 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1,
1028 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT,
1029 pmd->owner_uid, pmd->owner_gid);
1030 if (socket == NULL)
1031 return -1;
1032 ret = rte_hash_add_key_data(hash, key, socket);
1033 if (ret < 0) {
1034 MIF_LOG(ERR, "Failed to add socket to socket hash.");
1035 return ret;
1036 }
1037 }
1038 pmd->socket_filename = socket->filename;
1039
1040 TAILQ_FOREACH(elt, &socket->dev_queue, next) {
1041 tmp_pmd = elt->dev->data->dev_private;
1042 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
1043 MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
1044 "not have the same role.", pmd->id);
1045 return -1;
1046 }
1047 }
1048
1049 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
1050 if (elt == NULL) {
1051 MIF_LOG(ERR, "Failed to add device to socket device list.");
1052 return -1;
1053 }
1054 elt->dev = dev;
1055 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1056
1057 return 0;
1058 }
1059
1060 void
memif_socket_remove_device(struct rte_eth_dev * dev)1061 memif_socket_remove_device(struct rte_eth_dev *dev)
1062 {
1063 struct pmd_internals *pmd = dev->data->dev_private;
1064 struct memif_socket *socket = NULL;
1065 struct memif_socket_dev_list_elt *elt, *next;
1066 struct rte_hash *hash;
1067 int ret;
1068
1069 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1070 if (hash == NULL)
1071 return;
1072
1073 if (pmd->socket_filename == NULL)
1074 return;
1075
1076 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1077 return;
1078
1079 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1080 next = TAILQ_NEXT(elt, next);
1081 if (elt->dev == dev) {
1082 TAILQ_REMOVE(&socket->dev_queue, elt, next);
1083 rte_free(elt);
1084 pmd->socket_filename = NULL;
1085 }
1086 }
1087
1088 /* remove socket, if this was the last device using it */
1089 if (TAILQ_EMPTY(&socket->dev_queue)) {
1090 rte_hash_del_key(hash, socket->filename);
1091 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) {
1092 /* remove listener socket file,
1093 * so we can create new one later.
1094 */
1095 ret = remove(socket->filename);
1096 if (ret < 0)
1097 MIF_LOG(ERR, "Failed to remove socket file: %s",
1098 socket->filename);
1099 }
1100 if (pmd->role != MEMIF_ROLE_CLIENT)
1101 rte_intr_instance_free(socket->intr_handle);
1102 rte_free(socket);
1103 }
1104 }
1105
1106 int
memif_connect_server(struct rte_eth_dev * dev)1107 memif_connect_server(struct rte_eth_dev *dev)
1108 {
1109 struct pmd_internals *pmd = dev->data->dev_private;
1110
1111 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1112 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1113 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1114 return 0;
1115 }
1116
1117 int
memif_connect_client(struct rte_eth_dev * dev)1118 memif_connect_client(struct rte_eth_dev *dev)
1119 {
1120 int sockfd;
1121 int ret;
1122 uint32_t sunlen;
1123 struct sockaddr_un sun = { 0 };
1124 struct pmd_internals *pmd = dev->data->dev_private;
1125
1126 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1127 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1128 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1129
1130 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1131 if (sockfd < 0) {
1132 MIF_LOG(ERR, "Failed to open socket.");
1133 return -1;
1134 }
1135
1136 sun.sun_family = AF_UNIX;
1137 sunlen = sizeof(struct sockaddr_un);
1138 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) {
1139 /* abstract address */
1140 sun.sun_path[0] = '\0';
1141 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1);
1142 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1,
1143 MEMIF_SOCKET_UN_SIZE) +
1144 sizeof(sun) - sizeof(sun.sun_path);
1145 } else {
1146 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE);
1147 }
1148
1149 ret = connect(sockfd, (struct sockaddr *)&sun, sunlen);
1150 if (ret < 0) {
1151 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1152 goto error;
1153 }
1154
1155 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1156
1157 pmd->cc = rte_zmalloc("memif-cc",
1158 sizeof(struct memif_control_channel), 0);
1159 if (pmd->cc == NULL) {
1160 MIF_LOG(ERR, "Failed to allocate control channel.");
1161 goto error;
1162 }
1163
1164 /* Allocate interrupt instance */
1165 pmd->cc->intr_handle =
1166 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
1167 if (pmd->cc->intr_handle == NULL) {
1168 MIF_LOG(ERR, "Failed to allocate intr handle");
1169 goto error;
1170 }
1171
1172 if (rte_intr_fd_set(pmd->cc->intr_handle, sockfd))
1173 goto error;
1174
1175 if (rte_intr_type_set(pmd->cc->intr_handle, RTE_INTR_HANDLE_EXT))
1176 goto error;
1177
1178 pmd->cc->socket = NULL;
1179 pmd->cc->dev = dev;
1180 TAILQ_INIT(&pmd->cc->msg_queue);
1181
1182 ret = rte_intr_callback_register(pmd->cc->intr_handle,
1183 memif_intr_handler, pmd->cc);
1184 if (ret < 0) {
1185 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1186 goto error;
1187 }
1188
1189 return 0;
1190
1191 error:
1192 if (sockfd >= 0) {
1193 close(sockfd);
1194 sockfd = -1;
1195 }
1196 if (pmd->cc != NULL) {
1197 rte_intr_instance_free(pmd->cc->intr_handle);
1198 rte_free(pmd->cc);
1199 pmd->cc = NULL;
1200 }
1201 return -1;
1202 }
1203