1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/un.h> 11 #include <sys/ioctl.h> 12 #include <errno.h> 13 14 #include <rte_version.h> 15 #include <rte_mbuf.h> 16 #include <rte_ether.h> 17 #include <rte_ethdev_driver.h> 18 #include <rte_ethdev_vdev.h> 19 #include <rte_malloc.h> 20 #include <rte_kvargs.h> 21 #include <rte_bus_vdev.h> 22 #include <rte_hash.h> 23 #include <rte_jhash.h> 24 #include <rte_string_fns.h> 25 26 #include "rte_eth_memif.h" 27 #include "memif_socket.h" 28 29 static void memif_intr_handler(void *arg); 30 31 static ssize_t 32 memif_msg_send(int fd, memif_msg_t *msg, int afd) 33 { 34 struct msghdr mh = { 0 }; 35 struct iovec iov[1]; 36 struct cmsghdr *cmsg; 37 char ctl[CMSG_SPACE(sizeof(int))]; 38 39 iov[0].iov_base = msg; 40 iov[0].iov_len = sizeof(memif_msg_t); 41 mh.msg_iov = iov; 42 mh.msg_iovlen = 1; 43 44 if (afd > 0) { 45 memset(&ctl, 0, sizeof(ctl)); 46 mh.msg_control = ctl; 47 mh.msg_controllen = sizeof(ctl); 48 cmsg = CMSG_FIRSTHDR(&mh); 49 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 50 cmsg->cmsg_level = SOL_SOCKET; 51 cmsg->cmsg_type = SCM_RIGHTS; 52 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 53 } 54 55 return sendmsg(fd, &mh, 0); 56 } 57 58 static int 59 memif_msg_send_from_queue(struct memif_control_channel *cc) 60 { 61 ssize_t size; 62 int ret = 0; 63 struct memif_msg_queue_elt *e; 64 65 e = TAILQ_FIRST(&cc->msg_queue); 66 if (e == NULL) 67 return 0; 68 69 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd); 70 if (size != sizeof(memif_msg_t)) { 71 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 72 ret = -1; 73 } else { 74 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 75 } 76 TAILQ_REMOVE(&cc->msg_queue, e, next); 77 rte_free(e); 78 79 return ret; 80 } 81 82 static struct memif_msg_queue_elt * 83 memif_msg_enq(struct memif_control_channel *cc) 84 { 85 struct memif_msg_queue_elt *e; 86 87 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 88 if (e == NULL) { 89 MIF_LOG(ERR, "Failed to allocate control message."); 90 return NULL; 91 } 92 93 e->fd = -1; 94 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 95 96 return e; 97 } 98 99 void 100 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 101 int err_code) 102 { 103 struct memif_msg_queue_elt *e; 104 struct pmd_internals *pmd; 105 memif_msg_disconnect_t *d; 106 107 if (cc == NULL) { 108 MIF_LOG(DEBUG, "Missing control channel."); 109 return; 110 } 111 112 e = memif_msg_enq(cc); 113 if (e == NULL) { 114 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 115 return; 116 } 117 118 d = &e->msg.disconnect; 119 120 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 121 d->code = err_code; 122 123 if (reason != NULL) { 124 strlcpy((char *)d->string, reason, sizeof(d->string)); 125 if (cc->dev != NULL) { 126 pmd = cc->dev->data->dev_private; 127 strlcpy(pmd->local_disc_string, reason, 128 sizeof(pmd->local_disc_string)); 129 } 130 } 131 } 132 133 static int 134 memif_msg_enq_hello(struct memif_control_channel *cc) 135 { 136 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 137 memif_msg_hello_t *h; 138 139 if (e == NULL) 140 return -1; 141 142 h = &e->msg.hello; 143 144 e->msg.type = MEMIF_MSG_TYPE_HELLO; 145 h->min_version = MEMIF_VERSION; 146 h->max_version = MEMIF_VERSION; 147 h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 148 h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 149 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 150 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 151 152 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 153 154 return 0; 155 } 156 157 static int 158 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 159 { 160 struct pmd_internals *pmd = dev->data->dev_private; 161 memif_msg_hello_t *h = &msg->hello; 162 163 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 164 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 165 return -1; 166 } 167 168 /* Set parameters for active connection */ 169 pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1, 170 pmd->cfg.num_s2m_rings); 171 pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1, 172 pmd->cfg.num_m2s_rings); 173 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 174 pmd->cfg.log2_ring_size); 175 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 176 177 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 178 179 MIF_LOG(DEBUG, "%s: Connecting to %s.", 180 rte_vdev_device_name(pmd->vdev), pmd->remote_name); 181 182 return 0; 183 } 184 185 static int 186 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 187 { 188 memif_msg_init_t *i = &msg->init; 189 struct memif_socket_dev_list_elt *elt; 190 struct pmd_internals *pmd; 191 struct rte_eth_dev *dev; 192 193 if (i->version != MEMIF_VERSION) { 194 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 195 return -1; 196 } 197 198 if (cc->socket == NULL) { 199 memif_msg_enq_disconnect(cc, "Device error", 0); 200 return -1; 201 } 202 203 /* Find device with requested ID */ 204 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 205 dev = elt->dev; 206 pmd = dev->data->dev_private; 207 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 208 pmd->id == i->id) { 209 /* assign control channel to device */ 210 cc->dev = dev; 211 pmd->cc = cc; 212 213 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 214 memif_msg_enq_disconnect(pmd->cc, 215 "Only ethernet mode supported", 216 0); 217 return -1; 218 } 219 220 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 221 ETH_MEMIF_FLAG_CONNECTED)) { 222 memif_msg_enq_disconnect(pmd->cc, 223 "Already connected", 0); 224 return -1; 225 } 226 strlcpy(pmd->remote_name, (char *)i->name, 227 sizeof(pmd->remote_name)); 228 229 if (*pmd->secret != '\0') { 230 if (*i->secret == '\0') { 231 memif_msg_enq_disconnect(pmd->cc, 232 "Secret required", 0); 233 return -1; 234 } 235 if (strncmp(pmd->secret, (char *)i->secret, 236 ETH_MEMIF_SECRET_SIZE) != 0) { 237 memif_msg_enq_disconnect(pmd->cc, 238 "Incorrect secret", 0); 239 return -1; 240 } 241 } 242 243 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 244 return 0; 245 } 246 } 247 248 /* ID not found on this socket */ 249 MIF_LOG(DEBUG, "ID %u not found.", i->id); 250 memif_msg_enq_disconnect(cc, "ID not found", 0); 251 return -1; 252 } 253 254 static int 255 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 256 int fd) 257 { 258 struct pmd_internals *pmd = dev->data->dev_private; 259 struct pmd_process_private *proc_private = dev->process_private; 260 memif_msg_add_region_t *ar = &msg->add_region; 261 struct memif_region *r; 262 263 if (fd < 0) { 264 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 265 return -1; 266 } 267 268 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 269 ar->index != proc_private->regions_num || 270 proc_private->regions[ar->index] != NULL) { 271 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 272 return -1; 273 } 274 275 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 276 if (r == NULL) { 277 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 278 return -ENOMEM; 279 } 280 281 r->fd = fd; 282 r->region_size = ar->size; 283 r->addr = NULL; 284 285 proc_private->regions[ar->index] = r; 286 proc_private->regions_num++; 287 288 return 0; 289 } 290 291 static int 292 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 293 { 294 struct pmd_internals *pmd = dev->data->dev_private; 295 memif_msg_add_ring_t *ar = &msg->add_ring; 296 struct memif_queue *mq; 297 298 if (fd < 0) { 299 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 300 return -1; 301 } 302 303 /* check if we have enough queues */ 304 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) { 305 if (ar->index >= pmd->cfg.num_s2m_rings) { 306 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 307 return -1; 308 } 309 pmd->run.num_s2m_rings++; 310 } else { 311 if (ar->index >= pmd->cfg.num_m2s_rings) { 312 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 313 return -1; 314 } 315 pmd->run.num_m2s_rings++; 316 } 317 318 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? 319 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 320 321 mq->intr_handle.fd = fd; 322 mq->log2_ring_size = ar->log2_ring_size; 323 mq->region = ar->region; 324 mq->ring_offset = ar->offset; 325 326 return 0; 327 } 328 329 static int 330 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 331 { 332 struct pmd_internals *pmd = dev->data->dev_private; 333 memif_msg_connect_t *c = &msg->connect; 334 int ret; 335 336 ret = memif_connect(dev); 337 if (ret < 0) 338 return ret; 339 340 strlcpy(pmd->remote_if_name, (char *)c->if_name, 341 sizeof(pmd->remote_if_name)); 342 MIF_LOG(INFO, "%s: Remote interface %s connected.", 343 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name); 344 345 return 0; 346 } 347 348 static int 349 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 350 { 351 struct pmd_internals *pmd = dev->data->dev_private; 352 memif_msg_connected_t *c = &msg->connected; 353 int ret; 354 355 ret = memif_connect(dev); 356 if (ret < 0) 357 return ret; 358 359 strlcpy(pmd->remote_if_name, (char *)c->if_name, 360 sizeof(pmd->remote_if_name)); 361 MIF_LOG(INFO, "%s: Remote interface %s connected.", 362 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name); 363 364 return 0; 365 } 366 367 static int 368 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 369 { 370 struct pmd_internals *pmd = dev->data->dev_private; 371 memif_msg_disconnect_t *d = &msg->disconnect; 372 373 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 374 strlcpy(pmd->remote_disc_string, (char *)d->string, 375 sizeof(pmd->remote_disc_string)); 376 377 MIF_LOG(INFO, "%s: Disconnect received: %s", 378 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string); 379 380 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 381 memif_disconnect(dev); 382 return 0; 383 } 384 385 static int 386 memif_msg_enq_ack(struct rte_eth_dev *dev) 387 { 388 struct pmd_internals *pmd = dev->data->dev_private; 389 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 390 if (e == NULL) 391 return -1; 392 393 e->msg.type = MEMIF_MSG_TYPE_ACK; 394 395 return 0; 396 } 397 398 static int 399 memif_msg_enq_init(struct rte_eth_dev *dev) 400 { 401 struct pmd_internals *pmd = dev->data->dev_private; 402 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 403 memif_msg_init_t *i = &e->msg.init; 404 405 if (e == NULL) 406 return -1; 407 408 i = &e->msg.init; 409 e->msg.type = MEMIF_MSG_TYPE_INIT; 410 i->version = MEMIF_VERSION; 411 i->id = pmd->id; 412 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 413 414 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 415 416 if (*pmd->secret != '\0') 417 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 418 419 return 0; 420 } 421 422 static int 423 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 424 { 425 struct pmd_internals *pmd = dev->data->dev_private; 426 struct pmd_process_private *proc_private = dev->process_private; 427 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 428 memif_msg_add_region_t *ar; 429 struct memif_region *mr = proc_private->regions[idx]; 430 431 if (e == NULL) 432 return -1; 433 434 ar = &e->msg.add_region; 435 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 436 e->fd = mr->fd; 437 ar->index = idx; 438 ar->size = mr->region_size; 439 440 return 0; 441 } 442 443 static int 444 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 445 memif_ring_type_t type) 446 { 447 struct pmd_internals *pmd = dev->data->dev_private; 448 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 449 struct memif_queue *mq; 450 memif_msg_add_ring_t *ar; 451 452 if (e == NULL) 453 return -1; 454 455 ar = &e->msg.add_ring; 456 mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] : 457 dev->data->rx_queues[idx]; 458 459 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 460 e->fd = mq->intr_handle.fd; 461 ar->index = idx; 462 ar->offset = mq->ring_offset; 463 ar->region = mq->region; 464 ar->log2_ring_size = mq->log2_ring_size; 465 ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0; 466 ar->private_hdr_size = 0; 467 468 return 0; 469 } 470 471 static int 472 memif_msg_enq_connect(struct rte_eth_dev *dev) 473 { 474 struct pmd_internals *pmd = dev->data->dev_private; 475 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 476 const char *name = rte_vdev_device_name(pmd->vdev); 477 memif_msg_connect_t *c; 478 479 if (e == NULL) 480 return -1; 481 482 c = &e->msg.connect; 483 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 484 strlcpy((char *)c->if_name, name, sizeof(c->if_name)); 485 486 return 0; 487 } 488 489 static int 490 memif_msg_enq_connected(struct rte_eth_dev *dev) 491 { 492 struct pmd_internals *pmd = dev->data->dev_private; 493 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 494 const char *name = rte_vdev_device_name(pmd->vdev); 495 memif_msg_connected_t *c; 496 497 if (e == NULL) 498 return -1; 499 500 c = &e->msg.connected; 501 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 502 strlcpy((char *)c->if_name, name, sizeof(c->if_name)); 503 504 return 0; 505 } 506 507 static void 508 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 509 { 510 struct memif_msg_queue_elt *elt; 511 struct memif_control_channel *cc = arg; 512 513 /* close control channel fd */ 514 close(intr_handle->fd); 515 /* clear message queue */ 516 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 517 TAILQ_REMOVE(&cc->msg_queue, elt, next); 518 rte_free(elt); 519 } 520 /* free control channel */ 521 rte_free(cc); 522 } 523 524 void 525 memif_disconnect(struct rte_eth_dev *dev) 526 { 527 struct pmd_internals *pmd = dev->data->dev_private; 528 struct pmd_process_private *proc_private = dev->process_private; 529 struct memif_msg_queue_elt *elt, *next; 530 struct memif_queue *mq; 531 struct rte_intr_handle *ih; 532 int i; 533 int ret; 534 535 dev->data->dev_link.link_status = ETH_LINK_DOWN; 536 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 537 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 538 539 if (pmd->cc != NULL) { 540 /* Clear control message queue (except disconnect message if any). */ 541 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 542 next = TAILQ_NEXT(elt, next); 543 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 544 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 545 rte_free(elt); 546 } 547 } 548 /* send disconnect message (if there is any in queue) */ 549 memif_msg_send_from_queue(pmd->cc); 550 551 /* at this point, there should be no more messages in queue */ 552 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 553 MIF_LOG(WARNING, 554 "Unexpected message(s) in message queue."); 555 } 556 557 ih = &pmd->cc->intr_handle; 558 if (ih->fd > 0) { 559 ret = rte_intr_callback_unregister(ih, 560 memif_intr_handler, 561 pmd->cc); 562 /* 563 * If callback is active (disconnecting based on 564 * received control message). 565 */ 566 if (ret == -EAGAIN) { 567 ret = rte_intr_callback_unregister_pending(ih, 568 memif_intr_handler, 569 pmd->cc, 570 memif_intr_unregister_handler); 571 } else if (ret > 0) { 572 close(ih->fd); 573 rte_free(pmd->cc); 574 } 575 pmd->cc = NULL; 576 if (ret <= 0) 577 MIF_LOG(WARNING, 578 "Failed to unregister control channel callback."); 579 } 580 } 581 582 /* unconfig interrupts */ 583 for (i = 0; i < pmd->cfg.num_s2m_rings; i++) { 584 if (pmd->role == MEMIF_ROLE_SLAVE) { 585 if (dev->data->tx_queues != NULL) 586 mq = dev->data->tx_queues[i]; 587 else 588 continue; 589 } else { 590 if (dev->data->rx_queues != NULL) 591 mq = dev->data->rx_queues[i]; 592 else 593 continue; 594 } 595 if (mq->intr_handle.fd > 0) { 596 close(mq->intr_handle.fd); 597 mq->intr_handle.fd = -1; 598 } 599 } 600 for (i = 0; i < pmd->cfg.num_m2s_rings; i++) { 601 if (pmd->role == MEMIF_ROLE_MASTER) { 602 if (dev->data->tx_queues != NULL) 603 mq = dev->data->tx_queues[i]; 604 else 605 continue; 606 } else { 607 if (dev->data->rx_queues != NULL) 608 mq = dev->data->rx_queues[i]; 609 else 610 continue; 611 } 612 if (mq->intr_handle.fd > 0) { 613 close(mq->intr_handle.fd); 614 mq->intr_handle.fd = -1; 615 } 616 } 617 618 memif_free_regions(proc_private); 619 620 /* reset connection configuration */ 621 memset(&pmd->run, 0, sizeof(pmd->run)); 622 623 MIF_LOG(DEBUG, "Disconnected."); 624 } 625 626 static int 627 memif_msg_receive(struct memif_control_channel *cc) 628 { 629 char ctl[CMSG_SPACE(sizeof(int)) + 630 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 631 struct msghdr mh = { 0 }; 632 struct iovec iov[1]; 633 memif_msg_t msg = { 0 }; 634 ssize_t size; 635 int ret = 0; 636 struct ucred *cr __rte_unused; 637 cr = 0; 638 struct cmsghdr *cmsg; 639 int afd = -1; 640 int i; 641 struct pmd_internals *pmd; 642 struct pmd_process_private *proc_private; 643 644 iov[0].iov_base = (void *)&msg; 645 iov[0].iov_len = sizeof(memif_msg_t); 646 mh.msg_iov = iov; 647 mh.msg_iovlen = 1; 648 mh.msg_control = ctl; 649 mh.msg_controllen = sizeof(ctl); 650 651 size = recvmsg(cc->intr_handle.fd, &mh, 0); 652 if (size != sizeof(memif_msg_t)) { 653 MIF_LOG(DEBUG, "Invalid message size."); 654 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 655 return -1; 656 } 657 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 658 659 cmsg = CMSG_FIRSTHDR(&mh); 660 while (cmsg) { 661 if (cmsg->cmsg_level == SOL_SOCKET) { 662 if (cmsg->cmsg_type == SCM_CREDENTIALS) 663 cr = (struct ucred *)CMSG_DATA(cmsg); 664 else if (cmsg->cmsg_type == SCM_RIGHTS) 665 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 666 } 667 cmsg = CMSG_NXTHDR(&mh, cmsg); 668 } 669 670 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 671 MIF_LOG(DEBUG, "Unexpected message."); 672 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 673 return -1; 674 } 675 676 /* get device from hash data */ 677 switch (msg.type) { 678 case MEMIF_MSG_TYPE_ACK: 679 break; 680 case MEMIF_MSG_TYPE_HELLO: 681 ret = memif_msg_receive_hello(cc->dev, &msg); 682 if (ret < 0) 683 goto exit; 684 ret = memif_init_regions_and_queues(cc->dev); 685 if (ret < 0) 686 goto exit; 687 ret = memif_msg_enq_init(cc->dev); 688 if (ret < 0) 689 goto exit; 690 pmd = cc->dev->data->dev_private; 691 proc_private = cc->dev->process_private; 692 for (i = 0; i < proc_private->regions_num; i++) { 693 ret = memif_msg_enq_add_region(cc->dev, i); 694 if (ret < 0) 695 goto exit; 696 } 697 for (i = 0; i < pmd->run.num_s2m_rings; i++) { 698 ret = memif_msg_enq_add_ring(cc->dev, i, 699 MEMIF_RING_S2M); 700 if (ret < 0) 701 goto exit; 702 } 703 for (i = 0; i < pmd->run.num_m2s_rings; i++) { 704 ret = memif_msg_enq_add_ring(cc->dev, i, 705 MEMIF_RING_M2S); 706 if (ret < 0) 707 goto exit; 708 } 709 ret = memif_msg_enq_connect(cc->dev); 710 if (ret < 0) 711 goto exit; 712 break; 713 case MEMIF_MSG_TYPE_INIT: 714 /* 715 * This cc does not have an interface asociated with it. 716 * If suitable interface is found it will be assigned here. 717 */ 718 ret = memif_msg_receive_init(cc, &msg); 719 if (ret < 0) 720 goto exit; 721 ret = memif_msg_enq_ack(cc->dev); 722 if (ret < 0) 723 goto exit; 724 break; 725 case MEMIF_MSG_TYPE_ADD_REGION: 726 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 727 if (ret < 0) 728 goto exit; 729 ret = memif_msg_enq_ack(cc->dev); 730 if (ret < 0) 731 goto exit; 732 break; 733 case MEMIF_MSG_TYPE_ADD_RING: 734 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 735 if (ret < 0) 736 goto exit; 737 ret = memif_msg_enq_ack(cc->dev); 738 if (ret < 0) 739 goto exit; 740 break; 741 case MEMIF_MSG_TYPE_CONNECT: 742 ret = memif_msg_receive_connect(cc->dev, &msg); 743 if (ret < 0) 744 goto exit; 745 ret = memif_msg_enq_connected(cc->dev); 746 if (ret < 0) 747 goto exit; 748 break; 749 case MEMIF_MSG_TYPE_CONNECTED: 750 ret = memif_msg_receive_connected(cc->dev, &msg); 751 break; 752 case MEMIF_MSG_TYPE_DISCONNECT: 753 ret = memif_msg_receive_disconnect(cc->dev, &msg); 754 if (ret < 0) 755 goto exit; 756 break; 757 default: 758 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 759 ret = -1; 760 goto exit; 761 } 762 763 exit: 764 return ret; 765 } 766 767 static void 768 memif_intr_handler(void *arg) 769 { 770 struct memif_control_channel *cc = arg; 771 int ret; 772 773 ret = memif_msg_receive(cc); 774 /* if driver failed to assign device */ 775 if (cc->dev == NULL) { 776 ret = rte_intr_callback_unregister_pending(&cc->intr_handle, 777 memif_intr_handler, 778 cc, 779 memif_intr_unregister_handler); 780 if (ret < 0) 781 MIF_LOG(WARNING, 782 "Failed to unregister control channel callback."); 783 return; 784 } 785 /* if memif_msg_receive failed */ 786 if (ret < 0) 787 goto disconnect; 788 789 ret = memif_msg_send_from_queue(cc); 790 if (ret < 0) 791 goto disconnect; 792 793 return; 794 795 disconnect: 796 if (cc->dev == NULL) { 797 MIF_LOG(WARNING, "eth dev not allocated"); 798 return; 799 } 800 memif_disconnect(cc->dev); 801 } 802 803 static void 804 memif_listener_handler(void *arg) 805 { 806 struct memif_socket *socket = arg; 807 int sockfd; 808 int addr_len; 809 struct sockaddr_un client; 810 struct memif_control_channel *cc; 811 int ret; 812 813 addr_len = sizeof(client); 814 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client, 815 (socklen_t *)&addr_len); 816 if (sockfd < 0) { 817 MIF_LOG(ERR, 818 "Failed to accept connection request on socket fd %d", 819 socket->intr_handle.fd); 820 return; 821 } 822 823 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 824 825 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 826 if (cc == NULL) { 827 MIF_LOG(ERR, "Failed to allocate control channel."); 828 goto error; 829 } 830 831 cc->intr_handle.fd = sockfd; 832 cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 833 cc->socket = socket; 834 cc->dev = NULL; 835 TAILQ_INIT(&cc->msg_queue); 836 837 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc); 838 if (ret < 0) { 839 MIF_LOG(ERR, "Failed to register control channel callback."); 840 goto error; 841 } 842 843 ret = memif_msg_enq_hello(cc); 844 if (ret < 0) { 845 MIF_LOG(ERR, "Failed to enqueue hello message."); 846 goto error; 847 } 848 ret = memif_msg_send_from_queue(cc); 849 if (ret < 0) 850 goto error; 851 852 return; 853 854 error: 855 if (sockfd >= 0) { 856 close(sockfd); 857 sockfd = -1; 858 } 859 if (cc != NULL) 860 rte_free(cc); 861 } 862 863 #define MEMIF_SOCKET_UN_SIZE \ 864 (offsetof(struct sockaddr_un, sun_path) + MEMIF_SOCKET_KEY_LEN) 865 866 static struct memif_socket * 867 memif_socket_create(struct pmd_internals *pmd, 868 const char *key, uint8_t listener) 869 { 870 struct memif_socket *sock; 871 struct sockaddr_un *un; 872 char un_buf[MEMIF_SOCKET_UN_SIZE]; 873 int sockfd; 874 int ret; 875 int on = 1; 876 877 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 878 if (sock == NULL) { 879 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 880 return NULL; 881 } 882 883 sock->listener = listener; 884 strlcpy(sock->filename, key, MEMIF_SOCKET_KEY_LEN); 885 TAILQ_INIT(&sock->dev_queue); 886 887 if (listener != 0) { 888 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 889 if (sockfd < 0) 890 goto error; 891 892 memset(un_buf, 0, sizeof(un_buf)); 893 un = (struct sockaddr_un *)un_buf; 894 un->sun_family = AF_UNIX; 895 strlcpy(un->sun_path, sock->filename, MEMIF_SOCKET_KEY_LEN); 896 897 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 898 sizeof(on)); 899 if (ret < 0) 900 goto error; 901 ret = bind(sockfd, (struct sockaddr *)un, MEMIF_SOCKET_UN_SIZE); 902 if (ret < 0) 903 goto error; 904 ret = listen(sockfd, 1); 905 if (ret < 0) 906 goto error; 907 908 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.", 909 rte_vdev_device_name(pmd->vdev), sock->filename); 910 911 sock->intr_handle.fd = sockfd; 912 sock->intr_handle.type = RTE_INTR_HANDLE_EXT; 913 ret = rte_intr_callback_register(&sock->intr_handle, 914 memif_listener_handler, sock); 915 if (ret < 0) { 916 MIF_LOG(ERR, "%s: Failed to register interrupt " 917 "callback for listener socket", 918 rte_vdev_device_name(pmd->vdev)); 919 return NULL; 920 } 921 } 922 923 return sock; 924 925 error: 926 MIF_LOG(ERR, "%s: Failed to setup socket %s: %s", 927 rte_vdev_device_name(pmd->vdev) ? 928 rte_vdev_device_name(pmd->vdev) : "NULL", key, strerror(errno)); 929 if (sock != NULL) 930 rte_free(sock); 931 if (sockfd >= 0) 932 close(sockfd); 933 return NULL; 934 } 935 936 static struct rte_hash * 937 memif_create_socket_hash(void) 938 { 939 struct rte_hash_parameters params = { 0 }; 940 941 params.name = MEMIF_SOCKET_HASH_NAME; 942 params.entries = 256; 943 params.key_len = MEMIF_SOCKET_KEY_LEN; 944 params.hash_func = rte_jhash; 945 params.hash_func_init_val = 0; 946 return rte_hash_create(¶ms); 947 } 948 949 int 950 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 951 { 952 struct pmd_internals *pmd = dev->data->dev_private; 953 struct memif_socket *socket = NULL; 954 struct memif_socket_dev_list_elt *elt; 955 struct pmd_internals *tmp_pmd; 956 struct rte_hash *hash; 957 int ret; 958 char key[MEMIF_SOCKET_KEY_LEN]; 959 960 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 961 if (hash == NULL) { 962 hash = memif_create_socket_hash(); 963 if (hash == NULL) { 964 MIF_LOG(ERR, "Failed to create memif socket hash."); 965 return -1; 966 } 967 } 968 969 memset(key, 0, MEMIF_SOCKET_KEY_LEN); 970 strlcpy(key, socket_filename, MEMIF_SOCKET_KEY_LEN); 971 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 972 if (ret < 0) { 973 socket = memif_socket_create(pmd, key, 974 (pmd->role == 975 MEMIF_ROLE_SLAVE) ? 0 : 1); 976 if (socket == NULL) 977 return -1; 978 ret = rte_hash_add_key_data(hash, key, socket); 979 if (ret < 0) { 980 MIF_LOG(ERR, "Failed to add socket to socket hash."); 981 return ret; 982 } 983 } 984 pmd->socket_filename = socket->filename; 985 986 if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) { 987 MIF_LOG(ERR, "Socket is a listener."); 988 return -1; 989 } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) { 990 MIF_LOG(ERR, "Socket is not a listener."); 991 return -1; 992 } 993 994 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 995 tmp_pmd = elt->dev->data->dev_private; 996 if (tmp_pmd->id == pmd->id) { 997 MIF_LOG(ERR, "Memif device with id %d already " 998 "exists on socket %s", 999 pmd->id, socket->filename); 1000 return -1; 1001 } 1002 } 1003 1004 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 1005 if (elt == NULL) { 1006 MIF_LOG(ERR, "%s: Failed to add device to socket device list.", 1007 rte_vdev_device_name(pmd->vdev)); 1008 return -1; 1009 } 1010 elt->dev = dev; 1011 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 1012 1013 return 0; 1014 } 1015 1016 void 1017 memif_socket_remove_device(struct rte_eth_dev *dev) 1018 { 1019 struct pmd_internals *pmd = dev->data->dev_private; 1020 struct memif_socket *socket = NULL; 1021 struct memif_socket_dev_list_elt *elt, *next; 1022 struct rte_hash *hash; 1023 int ret; 1024 1025 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1026 if (hash == NULL) 1027 return; 1028 1029 if (pmd->socket_filename == NULL) 1030 return; 1031 1032 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1033 return; 1034 1035 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1036 next = TAILQ_NEXT(elt, next); 1037 if (elt->dev == dev) { 1038 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1039 rte_free(elt); 1040 pmd->socket_filename = NULL; 1041 } 1042 } 1043 1044 /* remove socket, if this was the last device using it */ 1045 if (TAILQ_EMPTY(&socket->dev_queue)) { 1046 rte_hash_del_key(hash, socket->filename); 1047 if (socket->listener) { 1048 /* remove listener socket file, 1049 * so we can create new one later. 1050 */ 1051 ret = remove(socket->filename); 1052 if (ret < 0) 1053 MIF_LOG(ERR, "Failed to remove socket file: %s", 1054 socket->filename); 1055 } 1056 rte_free(socket); 1057 } 1058 } 1059 1060 int 1061 memif_connect_master(struct rte_eth_dev *dev) 1062 { 1063 struct pmd_internals *pmd = dev->data->dev_private; 1064 1065 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1066 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1067 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1068 return 0; 1069 } 1070 1071 int 1072 memif_connect_slave(struct rte_eth_dev *dev) 1073 { 1074 int sockfd; 1075 int ret; 1076 struct sockaddr_un sun; 1077 struct pmd_internals *pmd = dev->data->dev_private; 1078 1079 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1080 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1081 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1082 1083 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1084 if (sockfd < 0) { 1085 MIF_LOG(ERR, "%s: Failed to open socket.", 1086 rte_vdev_device_name(pmd->vdev)); 1087 return -1; 1088 } 1089 1090 sun.sun_family = AF_UNIX; 1091 1092 memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1); 1093 1094 ret = connect(sockfd, (struct sockaddr *)&sun, 1095 sizeof(struct sockaddr_un)); 1096 if (ret < 0) { 1097 MIF_LOG(ERR, "%s: Failed to connect socket: %s.", 1098 rte_vdev_device_name(pmd->vdev), pmd->socket_filename); 1099 goto error; 1100 } 1101 1102 MIF_LOG(DEBUG, "%s: Memif socket: %s connected.", 1103 rte_vdev_device_name(pmd->vdev), pmd->socket_filename); 1104 1105 pmd->cc = rte_zmalloc("memif-cc", 1106 sizeof(struct memif_control_channel), 0); 1107 if (pmd->cc == NULL) { 1108 MIF_LOG(ERR, "%s: Failed to allocate control channel.", 1109 rte_vdev_device_name(pmd->vdev)); 1110 goto error; 1111 } 1112 1113 pmd->cc->intr_handle.fd = sockfd; 1114 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 1115 pmd->cc->socket = NULL; 1116 pmd->cc->dev = dev; 1117 TAILQ_INIT(&pmd->cc->msg_queue); 1118 1119 ret = rte_intr_callback_register(&pmd->cc->intr_handle, 1120 memif_intr_handler, pmd->cc); 1121 if (ret < 0) { 1122 MIF_LOG(ERR, "%s: Failed to register interrupt callback " 1123 "for control fd", rte_vdev_device_name(pmd->vdev)); 1124 goto error; 1125 } 1126 1127 return 0; 1128 1129 error: 1130 if (sockfd >= 0) { 1131 close(sockfd); 1132 sockfd = -1; 1133 } 1134 if (pmd->cc != NULL) { 1135 rte_free(pmd->cc); 1136 pmd->cc = NULL; 1137 } 1138 return -1; 1139 } 1140