1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/ioctl.h> 11 #include <errno.h> 12 13 #include <rte_version.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_ethdev_driver.h> 17 #include <rte_ethdev_vdev.h> 18 #include <rte_malloc.h> 19 #include <rte_kvargs.h> 20 #include <rte_bus_vdev.h> 21 #include <rte_hash.h> 22 #include <rte_jhash.h> 23 #include <rte_string_fns.h> 24 25 #include "rte_eth_memif.h" 26 #include "memif_socket.h" 27 28 static void memif_intr_handler(void *arg); 29 30 static ssize_t 31 memif_msg_send(int fd, memif_msg_t *msg, int afd) 32 { 33 struct msghdr mh = { 0 }; 34 struct iovec iov[1]; 35 struct cmsghdr *cmsg; 36 char ctl[CMSG_SPACE(sizeof(int))]; 37 38 iov[0].iov_base = msg; 39 iov[0].iov_len = sizeof(memif_msg_t); 40 mh.msg_iov = iov; 41 mh.msg_iovlen = 1; 42 43 if (afd > 0) { 44 memset(&ctl, 0, sizeof(ctl)); 45 mh.msg_control = ctl; 46 mh.msg_controllen = sizeof(ctl); 47 cmsg = CMSG_FIRSTHDR(&mh); 48 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 49 cmsg->cmsg_level = SOL_SOCKET; 50 cmsg->cmsg_type = SCM_RIGHTS; 51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 52 } 53 54 return sendmsg(fd, &mh, 0); 55 } 56 57 static int 58 memif_msg_send_from_queue(struct memif_control_channel *cc) 59 { 60 ssize_t size; 61 int ret = 0; 62 struct memif_msg_queue_elt *e; 63 64 e = TAILQ_FIRST(&cc->msg_queue); 65 if (e == NULL) 66 return 0; 67 68 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd); 69 if (size != sizeof(memif_msg_t)) { 70 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 71 ret = -1; 72 } else { 73 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 74 } 75 TAILQ_REMOVE(&cc->msg_queue, e, next); 76 rte_free(e); 77 78 return ret; 79 } 80 81 static struct memif_msg_queue_elt * 82 memif_msg_enq(struct memif_control_channel *cc) 83 { 84 struct memif_msg_queue_elt *e; 85 86 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 87 if (e == NULL) { 88 MIF_LOG(ERR, "Failed to allocate control message."); 89 return NULL; 90 } 91 92 e->fd = -1; 93 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 94 95 return e; 96 } 97 98 void 99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 100 int err_code) 101 { 102 struct memif_msg_queue_elt *e; 103 struct pmd_internals *pmd; 104 memif_msg_disconnect_t *d; 105 106 if (cc == NULL) { 107 MIF_LOG(DEBUG, "Missing control channel."); 108 return; 109 } 110 111 e = memif_msg_enq(cc); 112 if (e == NULL) { 113 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 114 return; 115 } 116 117 d = &e->msg.disconnect; 118 119 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 120 d->code = err_code; 121 122 if (reason != NULL) { 123 strlcpy((char *)d->string, reason, sizeof(d->string)); 124 if (cc->dev != NULL) { 125 pmd = cc->dev->data->dev_private; 126 strlcpy(pmd->local_disc_string, reason, 127 sizeof(pmd->local_disc_string)); 128 } 129 } 130 } 131 132 static int 133 memif_msg_enq_hello(struct memif_control_channel *cc) 134 { 135 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 136 memif_msg_hello_t *h; 137 138 if (e == NULL) 139 return -1; 140 141 h = &e->msg.hello; 142 143 e->msg.type = MEMIF_MSG_TYPE_HELLO; 144 h->min_version = MEMIF_VERSION; 145 h->max_version = MEMIF_VERSION; 146 h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 147 h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 148 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 149 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 150 151 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 152 153 return 0; 154 } 155 156 static int 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 158 { 159 struct pmd_internals *pmd = dev->data->dev_private; 160 memif_msg_hello_t *h = &msg->hello; 161 162 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 163 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 164 return -1; 165 } 166 167 /* Set parameters for active connection */ 168 pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1, 169 pmd->cfg.num_s2m_rings); 170 pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1, 171 pmd->cfg.num_m2s_rings); 172 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 173 pmd->cfg.log2_ring_size); 174 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 175 176 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 177 178 MIF_LOG(DEBUG, "%s: Connecting to %s.", 179 rte_vdev_device_name(pmd->vdev), pmd->remote_name); 180 181 return 0; 182 } 183 184 static int 185 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 186 { 187 memif_msg_init_t *i = &msg->init; 188 struct memif_socket_dev_list_elt *elt; 189 struct pmd_internals *pmd; 190 struct rte_eth_dev *dev; 191 192 if (i->version != MEMIF_VERSION) { 193 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 194 return -1; 195 } 196 197 if (cc->socket == NULL) { 198 memif_msg_enq_disconnect(cc, "Device error", 0); 199 return -1; 200 } 201 202 /* Find device with requested ID */ 203 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 204 dev = elt->dev; 205 pmd = dev->data->dev_private; 206 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 207 pmd->id == i->id) { 208 /* assign control channel to device */ 209 cc->dev = dev; 210 pmd->cc = cc; 211 212 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 213 memif_msg_enq_disconnect(pmd->cc, 214 "Only ethernet mode supported", 215 0); 216 return -1; 217 } 218 219 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 220 ETH_MEMIF_FLAG_CONNECTED)) { 221 memif_msg_enq_disconnect(pmd->cc, 222 "Already connected", 0); 223 return -1; 224 } 225 strlcpy(pmd->remote_name, (char *)i->name, 226 sizeof(pmd->remote_name)); 227 228 if (*pmd->secret != '\0') { 229 if (*i->secret == '\0') { 230 memif_msg_enq_disconnect(pmd->cc, 231 "Secret required", 0); 232 return -1; 233 } 234 if (strncmp(pmd->secret, (char *)i->secret, 235 ETH_MEMIF_SECRET_SIZE) != 0) { 236 memif_msg_enq_disconnect(pmd->cc, 237 "Incorrect secret", 0); 238 return -1; 239 } 240 } 241 242 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 243 return 0; 244 } 245 } 246 247 /* ID not found on this socket */ 248 MIF_LOG(DEBUG, "ID %u not found.", i->id); 249 memif_msg_enq_disconnect(cc, "ID not found", 0); 250 return -1; 251 } 252 253 static int 254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 255 int fd) 256 { 257 struct pmd_internals *pmd = dev->data->dev_private; 258 struct pmd_process_private *proc_private = dev->process_private; 259 memif_msg_add_region_t *ar = &msg->add_region; 260 struct memif_region *r; 261 262 if (fd < 0) { 263 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 264 return -1; 265 } 266 267 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 268 ar->index != proc_private->regions_num || 269 proc_private->regions[ar->index] != NULL) { 270 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 271 return -1; 272 } 273 274 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 275 if (r == NULL) { 276 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 277 return -ENOMEM; 278 } 279 280 r->fd = fd; 281 r->region_size = ar->size; 282 r->addr = NULL; 283 284 proc_private->regions[ar->index] = r; 285 proc_private->regions_num++; 286 287 return 0; 288 } 289 290 static int 291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 292 { 293 struct pmd_internals *pmd = dev->data->dev_private; 294 memif_msg_add_ring_t *ar = &msg->add_ring; 295 struct memif_queue *mq; 296 297 if (fd < 0) { 298 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 299 return -1; 300 } 301 302 /* check if we have enough queues */ 303 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) { 304 if (ar->index >= pmd->cfg.num_s2m_rings) { 305 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 306 return -1; 307 } 308 pmd->run.num_s2m_rings++; 309 } else { 310 if (ar->index >= pmd->cfg.num_m2s_rings) { 311 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 312 return -1; 313 } 314 pmd->run.num_m2s_rings++; 315 } 316 317 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? 318 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 319 320 mq->intr_handle.fd = fd; 321 mq->log2_ring_size = ar->log2_ring_size; 322 mq->region = ar->region; 323 mq->ring_offset = ar->offset; 324 325 return 0; 326 } 327 328 static int 329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 330 { 331 struct pmd_internals *pmd = dev->data->dev_private; 332 memif_msg_connect_t *c = &msg->connect; 333 int ret; 334 335 ret = memif_connect(dev); 336 if (ret < 0) 337 return ret; 338 339 strlcpy(pmd->remote_if_name, (char *)c->if_name, 340 sizeof(pmd->remote_if_name)); 341 MIF_LOG(INFO, "%s: Remote interface %s connected.", 342 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name); 343 344 return 0; 345 } 346 347 static int 348 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 349 { 350 struct pmd_internals *pmd = dev->data->dev_private; 351 memif_msg_connected_t *c = &msg->connected; 352 int ret; 353 354 ret = memif_connect(dev); 355 if (ret < 0) 356 return ret; 357 358 strlcpy(pmd->remote_if_name, (char *)c->if_name, 359 sizeof(pmd->remote_if_name)); 360 MIF_LOG(INFO, "%s: Remote interface %s connected.", 361 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name); 362 363 return 0; 364 } 365 366 static int 367 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 368 { 369 struct pmd_internals *pmd = dev->data->dev_private; 370 memif_msg_disconnect_t *d = &msg->disconnect; 371 372 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 373 strlcpy(pmd->remote_disc_string, (char *)d->string, 374 sizeof(pmd->remote_disc_string)); 375 376 MIF_LOG(INFO, "%s: Disconnect received: %s", 377 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string); 378 379 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 380 memif_disconnect(dev); 381 return 0; 382 } 383 384 static int 385 memif_msg_enq_ack(struct rte_eth_dev *dev) 386 { 387 struct pmd_internals *pmd = dev->data->dev_private; 388 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 389 if (e == NULL) 390 return -1; 391 392 e->msg.type = MEMIF_MSG_TYPE_ACK; 393 394 return 0; 395 } 396 397 static int 398 memif_msg_enq_init(struct rte_eth_dev *dev) 399 { 400 struct pmd_internals *pmd = dev->data->dev_private; 401 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 402 memif_msg_init_t *i = &e->msg.init; 403 404 if (e == NULL) 405 return -1; 406 407 i = &e->msg.init; 408 e->msg.type = MEMIF_MSG_TYPE_INIT; 409 i->version = MEMIF_VERSION; 410 i->id = pmd->id; 411 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 412 413 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 414 415 if (*pmd->secret != '\0') 416 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 417 418 return 0; 419 } 420 421 static int 422 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 423 { 424 struct pmd_internals *pmd = dev->data->dev_private; 425 struct pmd_process_private *proc_private = dev->process_private; 426 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 427 memif_msg_add_region_t *ar; 428 struct memif_region *mr = proc_private->regions[idx]; 429 430 if (e == NULL) 431 return -1; 432 433 ar = &e->msg.add_region; 434 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 435 e->fd = mr->fd; 436 ar->index = idx; 437 ar->size = mr->region_size; 438 439 return 0; 440 } 441 442 static int 443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 444 memif_ring_type_t type) 445 { 446 struct pmd_internals *pmd = dev->data->dev_private; 447 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 448 struct memif_queue *mq; 449 memif_msg_add_ring_t *ar; 450 451 if (e == NULL) 452 return -1; 453 454 ar = &e->msg.add_ring; 455 mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] : 456 dev->data->rx_queues[idx]; 457 458 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 459 e->fd = mq->intr_handle.fd; 460 ar->index = idx; 461 ar->offset = mq->ring_offset; 462 ar->region = mq->region; 463 ar->log2_ring_size = mq->log2_ring_size; 464 ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0; 465 ar->private_hdr_size = 0; 466 467 return 0; 468 } 469 470 static int 471 memif_msg_enq_connect(struct rte_eth_dev *dev) 472 { 473 struct pmd_internals *pmd = dev->data->dev_private; 474 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 475 const char *name = rte_vdev_device_name(pmd->vdev); 476 memif_msg_connect_t *c; 477 478 if (e == NULL) 479 return -1; 480 481 c = &e->msg.connect; 482 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 483 strlcpy((char *)c->if_name, name, sizeof(c->if_name)); 484 485 return 0; 486 } 487 488 static int 489 memif_msg_enq_connected(struct rte_eth_dev *dev) 490 { 491 struct pmd_internals *pmd = dev->data->dev_private; 492 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 493 const char *name = rte_vdev_device_name(pmd->vdev); 494 memif_msg_connected_t *c; 495 496 if (e == NULL) 497 return -1; 498 499 c = &e->msg.connected; 500 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 501 strlcpy((char *)c->if_name, name, sizeof(c->if_name)); 502 503 return 0; 504 } 505 506 static void 507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 508 { 509 struct memif_msg_queue_elt *elt; 510 struct memif_control_channel *cc = arg; 511 512 /* close control channel fd */ 513 close(intr_handle->fd); 514 /* clear message queue */ 515 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 516 TAILQ_REMOVE(&cc->msg_queue, elt, next); 517 rte_free(elt); 518 } 519 /* free control channel */ 520 rte_free(cc); 521 } 522 523 void 524 memif_disconnect(struct rte_eth_dev *dev) 525 { 526 struct pmd_internals *pmd = dev->data->dev_private; 527 struct pmd_process_private *proc_private = dev->process_private; 528 struct memif_msg_queue_elt *elt, *next; 529 struct memif_queue *mq; 530 struct rte_intr_handle *ih; 531 int i; 532 int ret; 533 534 dev->data->dev_link.link_status = ETH_LINK_DOWN; 535 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 536 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 537 538 if (pmd->cc != NULL) { 539 /* Clear control message queue (except disconnect message if any). */ 540 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 541 next = TAILQ_NEXT(elt, next); 542 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 543 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 544 rte_free(elt); 545 } 546 } 547 /* send disconnect message (if there is any in queue) */ 548 memif_msg_send_from_queue(pmd->cc); 549 550 /* at this point, there should be no more messages in queue */ 551 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 552 MIF_LOG(WARNING, 553 "Unexpected message(s) in message queue."); 554 } 555 556 ih = &pmd->cc->intr_handle; 557 if (ih->fd > 0) { 558 ret = rte_intr_callback_unregister(ih, 559 memif_intr_handler, 560 pmd->cc); 561 /* 562 * If callback is active (disconnecting based on 563 * received control message). 564 */ 565 if (ret == -EAGAIN) { 566 ret = rte_intr_callback_unregister_pending(ih, 567 memif_intr_handler, 568 pmd->cc, 569 memif_intr_unregister_handler); 570 } else if (ret > 0) { 571 close(ih->fd); 572 rte_free(pmd->cc); 573 } 574 pmd->cc = NULL; 575 if (ret <= 0) 576 MIF_LOG(WARNING, 577 "Failed to unregister control channel callback."); 578 } 579 } 580 581 /* unconfig interrupts */ 582 for (i = 0; i < pmd->cfg.num_s2m_rings; i++) { 583 if (pmd->role == MEMIF_ROLE_SLAVE) { 584 if (dev->data->tx_queues != NULL) 585 mq = dev->data->tx_queues[i]; 586 else 587 continue; 588 } else { 589 if (dev->data->rx_queues != NULL) 590 mq = dev->data->rx_queues[i]; 591 else 592 continue; 593 } 594 if (mq->intr_handle.fd > 0) { 595 close(mq->intr_handle.fd); 596 mq->intr_handle.fd = -1; 597 } 598 } 599 for (i = 0; i < pmd->cfg.num_m2s_rings; i++) { 600 if (pmd->role == MEMIF_ROLE_MASTER) { 601 if (dev->data->tx_queues != NULL) 602 mq = dev->data->tx_queues[i]; 603 else 604 continue; 605 } else { 606 if (dev->data->rx_queues != NULL) 607 mq = dev->data->rx_queues[i]; 608 else 609 continue; 610 } 611 if (mq->intr_handle.fd > 0) { 612 close(mq->intr_handle.fd); 613 mq->intr_handle.fd = -1; 614 } 615 } 616 617 memif_free_regions(proc_private); 618 619 /* reset connection configuration */ 620 memset(&pmd->run, 0, sizeof(pmd->run)); 621 622 MIF_LOG(DEBUG, "Disconnected."); 623 } 624 625 static int 626 memif_msg_receive(struct memif_control_channel *cc) 627 { 628 char ctl[CMSG_SPACE(sizeof(int)) + 629 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 630 struct msghdr mh = { 0 }; 631 struct iovec iov[1]; 632 memif_msg_t msg = { 0 }; 633 ssize_t size; 634 int ret = 0; 635 struct ucred *cr __rte_unused; 636 cr = 0; 637 struct cmsghdr *cmsg; 638 int afd = -1; 639 int i; 640 struct pmd_internals *pmd; 641 struct pmd_process_private *proc_private; 642 643 iov[0].iov_base = (void *)&msg; 644 iov[0].iov_len = sizeof(memif_msg_t); 645 mh.msg_iov = iov; 646 mh.msg_iovlen = 1; 647 mh.msg_control = ctl; 648 mh.msg_controllen = sizeof(ctl); 649 650 size = recvmsg(cc->intr_handle.fd, &mh, 0); 651 if (size != sizeof(memif_msg_t)) { 652 MIF_LOG(DEBUG, "Invalid message size."); 653 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 654 return -1; 655 } 656 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 657 658 cmsg = CMSG_FIRSTHDR(&mh); 659 while (cmsg) { 660 if (cmsg->cmsg_level == SOL_SOCKET) { 661 if (cmsg->cmsg_type == SCM_CREDENTIALS) 662 cr = (struct ucred *)CMSG_DATA(cmsg); 663 else if (cmsg->cmsg_type == SCM_RIGHTS) 664 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 665 } 666 cmsg = CMSG_NXTHDR(&mh, cmsg); 667 } 668 669 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 670 MIF_LOG(DEBUG, "Unexpected message."); 671 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 672 return -1; 673 } 674 675 /* get device from hash data */ 676 switch (msg.type) { 677 case MEMIF_MSG_TYPE_ACK: 678 break; 679 case MEMIF_MSG_TYPE_HELLO: 680 ret = memif_msg_receive_hello(cc->dev, &msg); 681 if (ret < 0) 682 goto exit; 683 ret = memif_init_regions_and_queues(cc->dev); 684 if (ret < 0) 685 goto exit; 686 ret = memif_msg_enq_init(cc->dev); 687 if (ret < 0) 688 goto exit; 689 pmd = cc->dev->data->dev_private; 690 proc_private = cc->dev->process_private; 691 for (i = 0; i < proc_private->regions_num; i++) { 692 ret = memif_msg_enq_add_region(cc->dev, i); 693 if (ret < 0) 694 goto exit; 695 } 696 for (i = 0; i < pmd->run.num_s2m_rings; i++) { 697 ret = memif_msg_enq_add_ring(cc->dev, i, 698 MEMIF_RING_S2M); 699 if (ret < 0) 700 goto exit; 701 } 702 for (i = 0; i < pmd->run.num_m2s_rings; i++) { 703 ret = memif_msg_enq_add_ring(cc->dev, i, 704 MEMIF_RING_M2S); 705 if (ret < 0) 706 goto exit; 707 } 708 ret = memif_msg_enq_connect(cc->dev); 709 if (ret < 0) 710 goto exit; 711 break; 712 case MEMIF_MSG_TYPE_INIT: 713 /* 714 * This cc does not have an interface asociated with it. 715 * If suitable interface is found it will be assigned here. 716 */ 717 ret = memif_msg_receive_init(cc, &msg); 718 if (ret < 0) 719 goto exit; 720 ret = memif_msg_enq_ack(cc->dev); 721 if (ret < 0) 722 goto exit; 723 break; 724 case MEMIF_MSG_TYPE_ADD_REGION: 725 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 726 if (ret < 0) 727 goto exit; 728 ret = memif_msg_enq_ack(cc->dev); 729 if (ret < 0) 730 goto exit; 731 break; 732 case MEMIF_MSG_TYPE_ADD_RING: 733 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 734 if (ret < 0) 735 goto exit; 736 ret = memif_msg_enq_ack(cc->dev); 737 if (ret < 0) 738 goto exit; 739 break; 740 case MEMIF_MSG_TYPE_CONNECT: 741 ret = memif_msg_receive_connect(cc->dev, &msg); 742 if (ret < 0) 743 goto exit; 744 ret = memif_msg_enq_connected(cc->dev); 745 if (ret < 0) 746 goto exit; 747 break; 748 case MEMIF_MSG_TYPE_CONNECTED: 749 ret = memif_msg_receive_connected(cc->dev, &msg); 750 break; 751 case MEMIF_MSG_TYPE_DISCONNECT: 752 ret = memif_msg_receive_disconnect(cc->dev, &msg); 753 if (ret < 0) 754 goto exit; 755 break; 756 default: 757 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 758 ret = -1; 759 goto exit; 760 } 761 762 exit: 763 return ret; 764 } 765 766 static void 767 memif_intr_handler(void *arg) 768 { 769 struct memif_control_channel *cc = arg; 770 int ret; 771 772 ret = memif_msg_receive(cc); 773 /* if driver failed to assign device */ 774 if (cc->dev == NULL) { 775 ret = rte_intr_callback_unregister_pending(&cc->intr_handle, 776 memif_intr_handler, 777 cc, 778 memif_intr_unregister_handler); 779 if (ret < 0) 780 MIF_LOG(WARNING, 781 "Failed to unregister control channel callback."); 782 return; 783 } 784 /* if memif_msg_receive failed */ 785 if (ret < 0) 786 goto disconnect; 787 788 ret = memif_msg_send_from_queue(cc); 789 if (ret < 0) 790 goto disconnect; 791 792 return; 793 794 disconnect: 795 if (cc->dev == NULL) { 796 MIF_LOG(WARNING, "eth dev not allocated"); 797 return; 798 } 799 memif_disconnect(cc->dev); 800 } 801 802 static void 803 memif_listener_handler(void *arg) 804 { 805 struct memif_socket *socket = arg; 806 int sockfd; 807 int addr_len; 808 struct sockaddr_un client; 809 struct memif_control_channel *cc; 810 int ret; 811 812 addr_len = sizeof(client); 813 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client, 814 (socklen_t *)&addr_len); 815 if (sockfd < 0) { 816 MIF_LOG(ERR, 817 "Failed to accept connection request on socket fd %d", 818 socket->intr_handle.fd); 819 return; 820 } 821 822 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 823 824 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 825 if (cc == NULL) { 826 MIF_LOG(ERR, "Failed to allocate control channel."); 827 goto error; 828 } 829 830 cc->intr_handle.fd = sockfd; 831 cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 832 cc->socket = socket; 833 cc->dev = NULL; 834 TAILQ_INIT(&cc->msg_queue); 835 836 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc); 837 if (ret < 0) { 838 MIF_LOG(ERR, "Failed to register control channel callback."); 839 goto error; 840 } 841 842 ret = memif_msg_enq_hello(cc); 843 if (ret < 0) { 844 MIF_LOG(ERR, "Failed to enqueue hello message."); 845 goto error; 846 } 847 ret = memif_msg_send_from_queue(cc); 848 if (ret < 0) 849 goto error; 850 851 return; 852 853 error: 854 if (sockfd >= 0) { 855 close(sockfd); 856 sockfd = -1; 857 } 858 if (cc != NULL) 859 rte_free(cc); 860 } 861 862 static struct memif_socket * 863 memif_socket_create(struct pmd_internals *pmd, 864 const char *key, uint8_t listener) 865 { 866 struct memif_socket *sock; 867 struct sockaddr_un un; 868 int sockfd; 869 int ret; 870 int on = 1; 871 872 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 873 if (sock == NULL) { 874 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 875 return NULL; 876 } 877 878 sock->listener = listener; 879 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE); 880 TAILQ_INIT(&sock->dev_queue); 881 882 if (listener != 0) { 883 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 884 if (sockfd < 0) 885 goto error; 886 887 un.sun_family = AF_UNIX; 888 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE); 889 890 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 891 sizeof(on)); 892 if (ret < 0) 893 goto error; 894 895 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un)); 896 if (ret < 0) 897 goto error; 898 899 ret = listen(sockfd, 1); 900 if (ret < 0) 901 goto error; 902 903 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.", 904 rte_vdev_device_name(pmd->vdev), sock->filename); 905 906 sock->intr_handle.fd = sockfd; 907 sock->intr_handle.type = RTE_INTR_HANDLE_EXT; 908 ret = rte_intr_callback_register(&sock->intr_handle, 909 memif_listener_handler, sock); 910 if (ret < 0) { 911 MIF_LOG(ERR, "%s: Failed to register interrupt " 912 "callback for listener socket", 913 rte_vdev_device_name(pmd->vdev)); 914 return NULL; 915 } 916 } 917 918 return sock; 919 920 error: 921 MIF_LOG(ERR, "%s: Failed to setup socket %s: %s", 922 rte_vdev_device_name(pmd->vdev) ? 923 rte_vdev_device_name(pmd->vdev) : "NULL", key, strerror(errno)); 924 if (sock != NULL) 925 rte_free(sock); 926 if (sockfd >= 0) 927 close(sockfd); 928 return NULL; 929 } 930 931 static struct rte_hash * 932 memif_create_socket_hash(void) 933 { 934 struct rte_hash_parameters params = { 0 }; 935 936 params.name = MEMIF_SOCKET_HASH_NAME; 937 params.entries = 256; 938 params.key_len = MEMIF_SOCKET_UN_SIZE; 939 params.hash_func = rte_jhash; 940 params.hash_func_init_val = 0; 941 return rte_hash_create(¶ms); 942 } 943 944 int 945 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 946 { 947 struct pmd_internals *pmd = dev->data->dev_private; 948 struct memif_socket *socket = NULL; 949 struct memif_socket_dev_list_elt *elt; 950 struct pmd_internals *tmp_pmd; 951 struct rte_hash *hash; 952 int ret; 953 char key[MEMIF_SOCKET_UN_SIZE]; 954 955 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 956 if (hash == NULL) { 957 hash = memif_create_socket_hash(); 958 if (hash == NULL) { 959 MIF_LOG(ERR, "Failed to create memif socket hash."); 960 return -1; 961 } 962 } 963 964 memset(key, 0, MEMIF_SOCKET_UN_SIZE); 965 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE); 966 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 967 if (ret < 0) { 968 socket = memif_socket_create(pmd, key, 969 (pmd->role == 970 MEMIF_ROLE_SLAVE) ? 0 : 1); 971 if (socket == NULL) 972 return -1; 973 ret = rte_hash_add_key_data(hash, key, socket); 974 if (ret < 0) { 975 MIF_LOG(ERR, "Failed to add socket to socket hash."); 976 return ret; 977 } 978 } 979 pmd->socket_filename = socket->filename; 980 981 if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) { 982 MIF_LOG(ERR, "Socket is a listener."); 983 return -1; 984 } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) { 985 MIF_LOG(ERR, "Socket is not a listener."); 986 return -1; 987 } 988 989 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 990 tmp_pmd = elt->dev->data->dev_private; 991 if (tmp_pmd->id == pmd->id) { 992 MIF_LOG(ERR, "Memif device with id %d already " 993 "exists on socket %s", 994 pmd->id, socket->filename); 995 return -1; 996 } 997 } 998 999 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 1000 if (elt == NULL) { 1001 MIF_LOG(ERR, "%s: Failed to add device to socket device list.", 1002 rte_vdev_device_name(pmd->vdev)); 1003 return -1; 1004 } 1005 elt->dev = dev; 1006 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 1007 1008 return 0; 1009 } 1010 1011 void 1012 memif_socket_remove_device(struct rte_eth_dev *dev) 1013 { 1014 struct pmd_internals *pmd = dev->data->dev_private; 1015 struct memif_socket *socket = NULL; 1016 struct memif_socket_dev_list_elt *elt, *next; 1017 struct rte_hash *hash; 1018 int ret; 1019 1020 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1021 if (hash == NULL) 1022 return; 1023 1024 if (pmd->socket_filename == NULL) 1025 return; 1026 1027 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1028 return; 1029 1030 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1031 next = TAILQ_NEXT(elt, next); 1032 if (elt->dev == dev) { 1033 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1034 rte_free(elt); 1035 pmd->socket_filename = NULL; 1036 } 1037 } 1038 1039 /* remove socket, if this was the last device using it */ 1040 if (TAILQ_EMPTY(&socket->dev_queue)) { 1041 rte_hash_del_key(hash, socket->filename); 1042 if (socket->listener) { 1043 /* remove listener socket file, 1044 * so we can create new one later. 1045 */ 1046 ret = remove(socket->filename); 1047 if (ret < 0) 1048 MIF_LOG(ERR, "Failed to remove socket file: %s", 1049 socket->filename); 1050 } 1051 rte_free(socket); 1052 } 1053 } 1054 1055 int 1056 memif_connect_master(struct rte_eth_dev *dev) 1057 { 1058 struct pmd_internals *pmd = dev->data->dev_private; 1059 1060 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1061 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1062 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1063 return 0; 1064 } 1065 1066 int 1067 memif_connect_slave(struct rte_eth_dev *dev) 1068 { 1069 int sockfd; 1070 int ret; 1071 struct sockaddr_un sun; 1072 struct pmd_internals *pmd = dev->data->dev_private; 1073 1074 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1075 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1076 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1077 1078 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1079 if (sockfd < 0) { 1080 MIF_LOG(ERR, "%s: Failed to open socket.", 1081 rte_vdev_device_name(pmd->vdev)); 1082 return -1; 1083 } 1084 1085 sun.sun_family = AF_UNIX; 1086 1087 memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1); 1088 1089 ret = connect(sockfd, (struct sockaddr *)&sun, 1090 sizeof(struct sockaddr_un)); 1091 if (ret < 0) { 1092 MIF_LOG(ERR, "%s: Failed to connect socket: %s.", 1093 rte_vdev_device_name(pmd->vdev), pmd->socket_filename); 1094 goto error; 1095 } 1096 1097 MIF_LOG(DEBUG, "%s: Memif socket: %s connected.", 1098 rte_vdev_device_name(pmd->vdev), pmd->socket_filename); 1099 1100 pmd->cc = rte_zmalloc("memif-cc", 1101 sizeof(struct memif_control_channel), 0); 1102 if (pmd->cc == NULL) { 1103 MIF_LOG(ERR, "%s: Failed to allocate control channel.", 1104 rte_vdev_device_name(pmd->vdev)); 1105 goto error; 1106 } 1107 1108 pmd->cc->intr_handle.fd = sockfd; 1109 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 1110 pmd->cc->socket = NULL; 1111 pmd->cc->dev = dev; 1112 TAILQ_INIT(&pmd->cc->msg_queue); 1113 1114 ret = rte_intr_callback_register(&pmd->cc->intr_handle, 1115 memif_intr_handler, pmd->cc); 1116 if (ret < 0) { 1117 MIF_LOG(ERR, "%s: Failed to register interrupt callback " 1118 "for control fd", rte_vdev_device_name(pmd->vdev)); 1119 goto error; 1120 } 1121 1122 return 0; 1123 1124 error: 1125 if (sockfd >= 0) { 1126 close(sockfd); 1127 sockfd = -1; 1128 } 1129 if (pmd->cc != NULL) { 1130 rte_free(pmd->cc); 1131 pmd->cc = NULL; 1132 } 1133 return -1; 1134 } 1135