1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/ioctl.h> 11 #include <errno.h> 12 13 #include <rte_version.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_ethdev_driver.h> 17 #include <rte_ethdev_vdev.h> 18 #include <rte_malloc.h> 19 #include <rte_kvargs.h> 20 #include <rte_bus_vdev.h> 21 #include <rte_hash.h> 22 #include <rte_jhash.h> 23 #include <rte_string_fns.h> 24 25 #include "rte_eth_memif.h" 26 #include "memif_socket.h" 27 28 static void memif_intr_handler(void *arg); 29 30 static ssize_t 31 memif_msg_send(int fd, memif_msg_t *msg, int afd) 32 { 33 struct msghdr mh = { 0 }; 34 struct iovec iov[1]; 35 struct cmsghdr *cmsg; 36 char ctl[CMSG_SPACE(sizeof(int))]; 37 38 iov[0].iov_base = msg; 39 iov[0].iov_len = sizeof(memif_msg_t); 40 mh.msg_iov = iov; 41 mh.msg_iovlen = 1; 42 43 if (afd > 0) { 44 memset(&ctl, 0, sizeof(ctl)); 45 mh.msg_control = ctl; 46 mh.msg_controllen = sizeof(ctl); 47 cmsg = CMSG_FIRSTHDR(&mh); 48 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 49 cmsg->cmsg_level = SOL_SOCKET; 50 cmsg->cmsg_type = SCM_RIGHTS; 51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 52 } 53 54 return sendmsg(fd, &mh, 0); 55 } 56 57 static int 58 memif_msg_send_from_queue(struct memif_control_channel *cc) 59 { 60 ssize_t size; 61 int ret = 0; 62 struct memif_msg_queue_elt *e; 63 64 e = TAILQ_FIRST(&cc->msg_queue); 65 if (e == NULL) 66 return 0; 67 68 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd); 69 if (size != sizeof(memif_msg_t)) { 70 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 71 ret = -1; 72 } else { 73 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 74 } 75 TAILQ_REMOVE(&cc->msg_queue, e, next); 76 rte_free(e); 77 78 return ret; 79 } 80 81 static struct memif_msg_queue_elt * 82 memif_msg_enq(struct memif_control_channel *cc) 83 { 84 struct memif_msg_queue_elt *e; 85 86 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 87 if (e == NULL) { 88 MIF_LOG(ERR, "Failed to allocate control message."); 89 return NULL; 90 } 91 92 e->fd = -1; 93 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 94 95 return e; 96 } 97 98 void 99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 100 int err_code) 101 { 102 struct memif_msg_queue_elt *e; 103 struct pmd_internals *pmd; 104 memif_msg_disconnect_t *d; 105 106 if (cc == NULL) { 107 MIF_LOG(DEBUG, "Missing control channel."); 108 return; 109 } 110 111 e = memif_msg_enq(cc); 112 if (e == NULL) { 113 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 114 return; 115 } 116 117 d = &e->msg.disconnect; 118 119 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 120 d->code = err_code; 121 122 if (reason != NULL) { 123 strlcpy((char *)d->string, reason, sizeof(d->string)); 124 if (cc->dev != NULL) { 125 pmd = cc->dev->data->dev_private; 126 strlcpy(pmd->local_disc_string, reason, 127 sizeof(pmd->local_disc_string)); 128 } 129 } 130 } 131 132 static int 133 memif_msg_enq_hello(struct memif_control_channel *cc) 134 { 135 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 136 memif_msg_hello_t *h; 137 138 if (e == NULL) 139 return -1; 140 141 h = &e->msg.hello; 142 143 e->msg.type = MEMIF_MSG_TYPE_HELLO; 144 h->min_version = MEMIF_VERSION; 145 h->max_version = MEMIF_VERSION; 146 h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 147 h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 148 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 149 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 150 151 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 152 153 return 0; 154 } 155 156 static int 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 158 { 159 struct pmd_internals *pmd = dev->data->dev_private; 160 memif_msg_hello_t *h = &msg->hello; 161 162 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 163 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 164 return -1; 165 } 166 167 /* Set parameters for active connection */ 168 pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1, 169 pmd->cfg.num_s2m_rings); 170 pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1, 171 pmd->cfg.num_m2s_rings); 172 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 173 pmd->cfg.log2_ring_size); 174 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 175 176 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 177 178 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name); 179 180 return 0; 181 } 182 183 static int 184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 185 { 186 memif_msg_init_t *i = &msg->init; 187 struct memif_socket_dev_list_elt *elt; 188 struct pmd_internals *pmd; 189 struct rte_eth_dev *dev; 190 191 if (i->version != MEMIF_VERSION) { 192 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 193 return -1; 194 } 195 196 if (cc->socket == NULL) { 197 memif_msg_enq_disconnect(cc, "Device error", 0); 198 return -1; 199 } 200 201 /* Find device with requested ID */ 202 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 203 dev = elt->dev; 204 pmd = dev->data->dev_private; 205 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 206 pmd->id == i->id) { 207 /* assign control channel to device */ 208 cc->dev = dev; 209 pmd->cc = cc; 210 211 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 212 memif_msg_enq_disconnect(pmd->cc, 213 "Only ethernet mode supported", 214 0); 215 return -1; 216 } 217 218 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 219 ETH_MEMIF_FLAG_CONNECTED)) { 220 memif_msg_enq_disconnect(pmd->cc, 221 "Already connected", 0); 222 return -1; 223 } 224 strlcpy(pmd->remote_name, (char *)i->name, 225 sizeof(pmd->remote_name)); 226 227 if (*pmd->secret != '\0') { 228 if (*i->secret == '\0') { 229 memif_msg_enq_disconnect(pmd->cc, 230 "Secret required", 0); 231 return -1; 232 } 233 if (strncmp(pmd->secret, (char *)i->secret, 234 ETH_MEMIF_SECRET_SIZE) != 0) { 235 memif_msg_enq_disconnect(pmd->cc, 236 "Incorrect secret", 0); 237 return -1; 238 } 239 } 240 241 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 242 return 0; 243 } 244 } 245 246 /* ID not found on this socket */ 247 MIF_LOG(DEBUG, "ID %u not found.", i->id); 248 memif_msg_enq_disconnect(cc, "ID not found", 0); 249 return -1; 250 } 251 252 static int 253 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 254 int fd) 255 { 256 struct pmd_internals *pmd = dev->data->dev_private; 257 struct pmd_process_private *proc_private = dev->process_private; 258 memif_msg_add_region_t *ar = &msg->add_region; 259 struct memif_region *r; 260 261 if (fd < 0) { 262 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 263 return -1; 264 } 265 266 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 267 ar->index != proc_private->regions_num || 268 proc_private->regions[ar->index] != NULL) { 269 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 270 return -1; 271 } 272 273 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 274 if (r == NULL) { 275 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 276 return -ENOMEM; 277 } 278 279 r->fd = fd; 280 r->region_size = ar->size; 281 r->addr = NULL; 282 283 proc_private->regions[ar->index] = r; 284 proc_private->regions_num++; 285 286 return 0; 287 } 288 289 static int 290 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 291 { 292 struct pmd_internals *pmd = dev->data->dev_private; 293 memif_msg_add_ring_t *ar = &msg->add_ring; 294 struct memif_queue *mq; 295 296 if (fd < 0) { 297 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 298 return -1; 299 } 300 301 /* check if we have enough queues */ 302 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) { 303 if (ar->index >= pmd->cfg.num_s2m_rings) { 304 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 305 return -1; 306 } 307 pmd->run.num_s2m_rings++; 308 } else { 309 if (ar->index >= pmd->cfg.num_m2s_rings) { 310 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 311 return -1; 312 } 313 pmd->run.num_m2s_rings++; 314 } 315 316 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? 317 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 318 319 mq->intr_handle.fd = fd; 320 mq->log2_ring_size = ar->log2_ring_size; 321 mq->region = ar->region; 322 mq->ring_offset = ar->offset; 323 324 return 0; 325 } 326 327 static int 328 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 329 { 330 struct pmd_internals *pmd = dev->data->dev_private; 331 memif_msg_connect_t *c = &msg->connect; 332 int ret; 333 334 ret = memif_connect(dev); 335 if (ret < 0) 336 return ret; 337 338 strlcpy(pmd->remote_if_name, (char *)c->if_name, 339 sizeof(pmd->remote_if_name)); 340 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 341 342 return 0; 343 } 344 345 static int 346 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 347 { 348 struct pmd_internals *pmd = dev->data->dev_private; 349 memif_msg_connected_t *c = &msg->connected; 350 int ret; 351 352 ret = memif_connect(dev); 353 if (ret < 0) 354 return ret; 355 356 strlcpy(pmd->remote_if_name, (char *)c->if_name, 357 sizeof(pmd->remote_if_name)); 358 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 359 360 return 0; 361 } 362 363 static int 364 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 365 { 366 struct pmd_internals *pmd = dev->data->dev_private; 367 memif_msg_disconnect_t *d = &msg->disconnect; 368 369 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string)); 370 strlcpy(pmd->remote_disc_string, (char *)d->string, 371 sizeof(pmd->remote_disc_string)); 372 373 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string); 374 375 memset(pmd->local_disc_string, 0, 96); 376 memif_disconnect(dev); 377 return 0; 378 } 379 380 static int 381 memif_msg_enq_ack(struct rte_eth_dev *dev) 382 { 383 struct pmd_internals *pmd = dev->data->dev_private; 384 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 385 if (e == NULL) 386 return -1; 387 388 e->msg.type = MEMIF_MSG_TYPE_ACK; 389 390 return 0; 391 } 392 393 static int 394 memif_msg_enq_init(struct rte_eth_dev *dev) 395 { 396 struct pmd_internals *pmd = dev->data->dev_private; 397 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 398 memif_msg_init_t *i = &e->msg.init; 399 400 if (e == NULL) 401 return -1; 402 403 i = &e->msg.init; 404 e->msg.type = MEMIF_MSG_TYPE_INIT; 405 i->version = MEMIF_VERSION; 406 i->id = pmd->id; 407 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 408 409 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 410 411 if (*pmd->secret != '\0') 412 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 413 414 return 0; 415 } 416 417 static int 418 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 419 { 420 struct pmd_internals *pmd = dev->data->dev_private; 421 struct pmd_process_private *proc_private = dev->process_private; 422 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 423 memif_msg_add_region_t *ar; 424 struct memif_region *mr = proc_private->regions[idx]; 425 426 if (e == NULL) 427 return -1; 428 429 ar = &e->msg.add_region; 430 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 431 e->fd = mr->fd; 432 ar->index = idx; 433 ar->size = mr->region_size; 434 435 return 0; 436 } 437 438 static int 439 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 440 memif_ring_type_t type) 441 { 442 struct pmd_internals *pmd = dev->data->dev_private; 443 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 444 struct memif_queue *mq; 445 memif_msg_add_ring_t *ar; 446 447 if (e == NULL) 448 return -1; 449 450 ar = &e->msg.add_ring; 451 mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] : 452 dev->data->rx_queues[idx]; 453 454 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 455 e->fd = mq->intr_handle.fd; 456 ar->index = idx; 457 ar->offset = mq->ring_offset; 458 ar->region = mq->region; 459 ar->log2_ring_size = mq->log2_ring_size; 460 ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0; 461 ar->private_hdr_size = 0; 462 463 return 0; 464 } 465 466 static int 467 memif_msg_enq_connect(struct rte_eth_dev *dev) 468 { 469 struct pmd_internals *pmd = dev->data->dev_private; 470 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 471 memif_msg_connect_t *c; 472 473 if (e == NULL) 474 return -1; 475 476 c = &e->msg.connect; 477 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 478 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 479 480 return 0; 481 } 482 483 static int 484 memif_msg_enq_connected(struct rte_eth_dev *dev) 485 { 486 struct pmd_internals *pmd = dev->data->dev_private; 487 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 488 memif_msg_connected_t *c; 489 490 if (e == NULL) 491 return -1; 492 493 c = &e->msg.connected; 494 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 495 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 496 497 return 0; 498 } 499 500 static void 501 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 502 { 503 struct memif_msg_queue_elt *elt; 504 struct memif_control_channel *cc = arg; 505 506 /* close control channel fd */ 507 close(intr_handle->fd); 508 /* clear message queue */ 509 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 510 TAILQ_REMOVE(&cc->msg_queue, elt, next); 511 rte_free(elt); 512 } 513 /* free control channel */ 514 rte_free(cc); 515 } 516 517 void 518 memif_disconnect(struct rte_eth_dev *dev) 519 { 520 struct pmd_internals *pmd = dev->data->dev_private; 521 struct memif_msg_queue_elt *elt, *next; 522 struct memif_queue *mq; 523 struct rte_intr_handle *ih; 524 int i; 525 int ret; 526 527 dev->data->dev_link.link_status = ETH_LINK_DOWN; 528 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 529 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 530 531 if (pmd->cc != NULL) { 532 /* Clear control message queue (except disconnect message if any). */ 533 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 534 next = TAILQ_NEXT(elt, next); 535 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 536 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 537 rte_free(elt); 538 } 539 } 540 /* send disconnect message (if there is any in queue) */ 541 memif_msg_send_from_queue(pmd->cc); 542 543 /* at this point, there should be no more messages in queue */ 544 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 545 MIF_LOG(WARNING, 546 "Unexpected message(s) in message queue."); 547 } 548 549 ih = &pmd->cc->intr_handle; 550 if (ih->fd > 0) { 551 ret = rte_intr_callback_unregister(ih, 552 memif_intr_handler, 553 pmd->cc); 554 /* 555 * If callback is active (disconnecting based on 556 * received control message). 557 */ 558 if (ret == -EAGAIN) { 559 ret = rte_intr_callback_unregister_pending(ih, 560 memif_intr_handler, 561 pmd->cc, 562 memif_intr_unregister_handler); 563 } else if (ret > 0) { 564 close(ih->fd); 565 rte_free(pmd->cc); 566 } 567 pmd->cc = NULL; 568 if (ret <= 0) 569 MIF_LOG(WARNING, 570 "Failed to unregister control channel callback."); 571 } 572 } 573 574 /* unconfig interrupts */ 575 for (i = 0; i < pmd->cfg.num_s2m_rings; i++) { 576 if (pmd->role == MEMIF_ROLE_SLAVE) { 577 if (dev->data->tx_queues != NULL) 578 mq = dev->data->tx_queues[i]; 579 else 580 continue; 581 } else { 582 if (dev->data->rx_queues != NULL) 583 mq = dev->data->rx_queues[i]; 584 else 585 continue; 586 } 587 if (mq->intr_handle.fd > 0) { 588 close(mq->intr_handle.fd); 589 mq->intr_handle.fd = -1; 590 } 591 } 592 for (i = 0; i < pmd->cfg.num_m2s_rings; i++) { 593 if (pmd->role == MEMIF_ROLE_MASTER) { 594 if (dev->data->tx_queues != NULL) 595 mq = dev->data->tx_queues[i]; 596 else 597 continue; 598 } else { 599 if (dev->data->rx_queues != NULL) 600 mq = dev->data->rx_queues[i]; 601 else 602 continue; 603 } 604 if (mq->intr_handle.fd > 0) { 605 close(mq->intr_handle.fd); 606 mq->intr_handle.fd = -1; 607 } 608 } 609 610 memif_free_regions(dev); 611 612 /* reset connection configuration */ 613 memset(&pmd->run, 0, sizeof(pmd->run)); 614 615 MIF_LOG(DEBUG, "Disconnected."); 616 } 617 618 static int 619 memif_msg_receive(struct memif_control_channel *cc) 620 { 621 char ctl[CMSG_SPACE(sizeof(int)) + 622 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 623 struct msghdr mh = { 0 }; 624 struct iovec iov[1]; 625 memif_msg_t msg = { 0 }; 626 ssize_t size; 627 int ret = 0; 628 struct ucred *cr __rte_unused; 629 cr = 0; 630 struct cmsghdr *cmsg; 631 int afd = -1; 632 int i; 633 struct pmd_internals *pmd; 634 struct pmd_process_private *proc_private; 635 636 iov[0].iov_base = (void *)&msg; 637 iov[0].iov_len = sizeof(memif_msg_t); 638 mh.msg_iov = iov; 639 mh.msg_iovlen = 1; 640 mh.msg_control = ctl; 641 mh.msg_controllen = sizeof(ctl); 642 643 size = recvmsg(cc->intr_handle.fd, &mh, 0); 644 if (size != sizeof(memif_msg_t)) { 645 MIF_LOG(DEBUG, "Invalid message size."); 646 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 647 return -1; 648 } 649 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 650 651 cmsg = CMSG_FIRSTHDR(&mh); 652 while (cmsg) { 653 if (cmsg->cmsg_level == SOL_SOCKET) { 654 if (cmsg->cmsg_type == SCM_CREDENTIALS) 655 cr = (struct ucred *)CMSG_DATA(cmsg); 656 else if (cmsg->cmsg_type == SCM_RIGHTS) 657 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 658 } 659 cmsg = CMSG_NXTHDR(&mh, cmsg); 660 } 661 662 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 663 MIF_LOG(DEBUG, "Unexpected message."); 664 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 665 return -1; 666 } 667 668 /* get device from hash data */ 669 switch (msg.type) { 670 case MEMIF_MSG_TYPE_ACK: 671 break; 672 case MEMIF_MSG_TYPE_HELLO: 673 ret = memif_msg_receive_hello(cc->dev, &msg); 674 if (ret < 0) 675 goto exit; 676 ret = memif_init_regions_and_queues(cc->dev); 677 if (ret < 0) 678 goto exit; 679 ret = memif_msg_enq_init(cc->dev); 680 if (ret < 0) 681 goto exit; 682 pmd = cc->dev->data->dev_private; 683 proc_private = cc->dev->process_private; 684 for (i = 0; i < proc_private->regions_num; i++) { 685 ret = memif_msg_enq_add_region(cc->dev, i); 686 if (ret < 0) 687 goto exit; 688 } 689 for (i = 0; i < pmd->run.num_s2m_rings; i++) { 690 ret = memif_msg_enq_add_ring(cc->dev, i, 691 MEMIF_RING_S2M); 692 if (ret < 0) 693 goto exit; 694 } 695 for (i = 0; i < pmd->run.num_m2s_rings; i++) { 696 ret = memif_msg_enq_add_ring(cc->dev, i, 697 MEMIF_RING_M2S); 698 if (ret < 0) 699 goto exit; 700 } 701 ret = memif_msg_enq_connect(cc->dev); 702 if (ret < 0) 703 goto exit; 704 break; 705 case MEMIF_MSG_TYPE_INIT: 706 /* 707 * This cc does not have an interface asociated with it. 708 * If suitable interface is found it will be assigned here. 709 */ 710 ret = memif_msg_receive_init(cc, &msg); 711 if (ret < 0) 712 goto exit; 713 ret = memif_msg_enq_ack(cc->dev); 714 if (ret < 0) 715 goto exit; 716 break; 717 case MEMIF_MSG_TYPE_ADD_REGION: 718 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 719 if (ret < 0) 720 goto exit; 721 ret = memif_msg_enq_ack(cc->dev); 722 if (ret < 0) 723 goto exit; 724 break; 725 case MEMIF_MSG_TYPE_ADD_RING: 726 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 727 if (ret < 0) 728 goto exit; 729 ret = memif_msg_enq_ack(cc->dev); 730 if (ret < 0) 731 goto exit; 732 break; 733 case MEMIF_MSG_TYPE_CONNECT: 734 ret = memif_msg_receive_connect(cc->dev, &msg); 735 if (ret < 0) 736 goto exit; 737 ret = memif_msg_enq_connected(cc->dev); 738 if (ret < 0) 739 goto exit; 740 break; 741 case MEMIF_MSG_TYPE_CONNECTED: 742 ret = memif_msg_receive_connected(cc->dev, &msg); 743 break; 744 case MEMIF_MSG_TYPE_DISCONNECT: 745 ret = memif_msg_receive_disconnect(cc->dev, &msg); 746 if (ret < 0) 747 goto exit; 748 break; 749 default: 750 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 751 ret = -1; 752 goto exit; 753 } 754 755 exit: 756 return ret; 757 } 758 759 static void 760 memif_intr_handler(void *arg) 761 { 762 struct memif_control_channel *cc = arg; 763 int ret; 764 765 ret = memif_msg_receive(cc); 766 /* if driver failed to assign device */ 767 if (cc->dev == NULL) { 768 ret = rte_intr_callback_unregister_pending(&cc->intr_handle, 769 memif_intr_handler, 770 cc, 771 memif_intr_unregister_handler); 772 if (ret < 0) 773 MIF_LOG(WARNING, 774 "Failed to unregister control channel callback."); 775 return; 776 } 777 /* if memif_msg_receive failed */ 778 if (ret < 0) 779 goto disconnect; 780 781 ret = memif_msg_send_from_queue(cc); 782 if (ret < 0) 783 goto disconnect; 784 785 return; 786 787 disconnect: 788 if (cc->dev == NULL) { 789 MIF_LOG(WARNING, "eth dev not allocated"); 790 return; 791 } 792 memif_disconnect(cc->dev); 793 } 794 795 static void 796 memif_listener_handler(void *arg) 797 { 798 struct memif_socket *socket = arg; 799 int sockfd; 800 int addr_len; 801 struct sockaddr_un client; 802 struct memif_control_channel *cc; 803 int ret; 804 805 addr_len = sizeof(client); 806 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client, 807 (socklen_t *)&addr_len); 808 if (sockfd < 0) { 809 MIF_LOG(ERR, 810 "Failed to accept connection request on socket fd %d", 811 socket->intr_handle.fd); 812 return; 813 } 814 815 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 816 817 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 818 if (cc == NULL) { 819 MIF_LOG(ERR, "Failed to allocate control channel."); 820 goto error; 821 } 822 823 cc->intr_handle.fd = sockfd; 824 cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 825 cc->socket = socket; 826 cc->dev = NULL; 827 TAILQ_INIT(&cc->msg_queue); 828 829 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc); 830 if (ret < 0) { 831 MIF_LOG(ERR, "Failed to register control channel callback."); 832 goto error; 833 } 834 835 ret = memif_msg_enq_hello(cc); 836 if (ret < 0) { 837 MIF_LOG(ERR, "Failed to enqueue hello message."); 838 goto error; 839 } 840 ret = memif_msg_send_from_queue(cc); 841 if (ret < 0) 842 goto error; 843 844 return; 845 846 error: 847 if (sockfd >= 0) { 848 close(sockfd); 849 sockfd = -1; 850 } 851 if (cc != NULL) 852 rte_free(cc); 853 } 854 855 static struct memif_socket * 856 memif_socket_create(char *key, uint8_t listener) 857 { 858 struct memif_socket *sock; 859 struct sockaddr_un un; 860 int sockfd; 861 int ret; 862 int on = 1; 863 864 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 865 if (sock == NULL) { 866 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 867 return NULL; 868 } 869 870 sock->listener = listener; 871 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE); 872 TAILQ_INIT(&sock->dev_queue); 873 874 if (listener != 0) { 875 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 876 if (sockfd < 0) 877 goto error; 878 879 un.sun_family = AF_UNIX; 880 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE); 881 882 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 883 sizeof(on)); 884 if (ret < 0) 885 goto error; 886 887 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un)); 888 if (ret < 0) 889 goto error; 890 891 ret = listen(sockfd, 1); 892 if (ret < 0) 893 goto error; 894 895 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename); 896 897 sock->intr_handle.fd = sockfd; 898 sock->intr_handle.type = RTE_INTR_HANDLE_EXT; 899 ret = rte_intr_callback_register(&sock->intr_handle, 900 memif_listener_handler, sock); 901 if (ret < 0) { 902 MIF_LOG(ERR, "Failed to register interrupt " 903 "callback for listener socket"); 904 return NULL; 905 } 906 } 907 908 return sock; 909 910 error: 911 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno)); 912 if (sock != NULL) 913 rte_free(sock); 914 if (sockfd >= 0) 915 close(sockfd); 916 return NULL; 917 } 918 919 static struct rte_hash * 920 memif_create_socket_hash(void) 921 { 922 struct rte_hash_parameters params = { 0 }; 923 924 params.name = MEMIF_SOCKET_HASH_NAME; 925 params.entries = 256; 926 params.key_len = MEMIF_SOCKET_UN_SIZE; 927 params.hash_func = rte_jhash; 928 params.hash_func_init_val = 0; 929 return rte_hash_create(¶ms); 930 } 931 932 int 933 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 934 { 935 struct pmd_internals *pmd = dev->data->dev_private; 936 struct memif_socket *socket = NULL; 937 struct memif_socket_dev_list_elt *elt; 938 struct pmd_internals *tmp_pmd; 939 struct rte_hash *hash; 940 int ret; 941 char key[MEMIF_SOCKET_UN_SIZE]; 942 943 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 944 if (hash == NULL) { 945 hash = memif_create_socket_hash(); 946 if (hash == NULL) { 947 MIF_LOG(ERR, "Failed to create memif socket hash."); 948 return -1; 949 } 950 } 951 952 memset(key, 0, MEMIF_SOCKET_UN_SIZE); 953 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE); 954 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 955 if (ret < 0) { 956 socket = memif_socket_create(key, 957 (pmd->role == MEMIF_ROLE_SLAVE) ? 0 : 1); 958 if (socket == NULL) 959 return -1; 960 ret = rte_hash_add_key_data(hash, key, socket); 961 if (ret < 0) { 962 MIF_LOG(ERR, "Failed to add socket to socket hash."); 963 return ret; 964 } 965 } 966 pmd->socket_filename = socket->filename; 967 968 if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) { 969 MIF_LOG(ERR, "Socket is a listener."); 970 return -1; 971 } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) { 972 MIF_LOG(ERR, "Socket is not a listener."); 973 return -1; 974 } 975 976 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 977 tmp_pmd = elt->dev->data->dev_private; 978 if (tmp_pmd->id == pmd->id) { 979 MIF_LOG(ERR, "Memif device with id %d already " 980 "exists on socket %s", 981 pmd->id, socket->filename); 982 return -1; 983 } 984 } 985 986 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 987 if (elt == NULL) { 988 MIF_LOG(ERR, "Failed to add device to socket device list."); 989 return -1; 990 } 991 elt->dev = dev; 992 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 993 994 return 0; 995 } 996 997 void 998 memif_socket_remove_device(struct rte_eth_dev *dev) 999 { 1000 struct pmd_internals *pmd = dev->data->dev_private; 1001 struct memif_socket *socket = NULL; 1002 struct memif_socket_dev_list_elt *elt, *next; 1003 struct rte_hash *hash; 1004 int ret; 1005 1006 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1007 if (hash == NULL) 1008 return; 1009 1010 if (pmd->socket_filename == NULL) 1011 return; 1012 1013 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1014 return; 1015 1016 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1017 next = TAILQ_NEXT(elt, next); 1018 if (elt->dev == dev) { 1019 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1020 rte_free(elt); 1021 pmd->socket_filename = NULL; 1022 } 1023 } 1024 1025 /* remove socket, if this was the last device using it */ 1026 if (TAILQ_EMPTY(&socket->dev_queue)) { 1027 rte_hash_del_key(hash, socket->filename); 1028 if (socket->listener) { 1029 /* remove listener socket file, 1030 * so we can create new one later. 1031 */ 1032 ret = remove(socket->filename); 1033 if (ret < 0) 1034 MIF_LOG(ERR, "Failed to remove socket file: %s", 1035 socket->filename); 1036 } 1037 rte_free(socket); 1038 } 1039 } 1040 1041 int 1042 memif_connect_master(struct rte_eth_dev *dev) 1043 { 1044 struct pmd_internals *pmd = dev->data->dev_private; 1045 1046 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1047 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1048 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1049 return 0; 1050 } 1051 1052 int 1053 memif_connect_slave(struct rte_eth_dev *dev) 1054 { 1055 int sockfd; 1056 int ret; 1057 struct sockaddr_un sun; 1058 struct pmd_internals *pmd = dev->data->dev_private; 1059 1060 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1061 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1062 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1063 1064 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1065 if (sockfd < 0) { 1066 MIF_LOG(ERR, "Failed to open socket."); 1067 return -1; 1068 } 1069 1070 sun.sun_family = AF_UNIX; 1071 1072 memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1); 1073 1074 ret = connect(sockfd, (struct sockaddr *)&sun, 1075 sizeof(struct sockaddr_un)); 1076 if (ret < 0) { 1077 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename); 1078 goto error; 1079 } 1080 1081 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename); 1082 1083 pmd->cc = rte_zmalloc("memif-cc", 1084 sizeof(struct memif_control_channel), 0); 1085 if (pmd->cc == NULL) { 1086 MIF_LOG(ERR, "Failed to allocate control channel."); 1087 goto error; 1088 } 1089 1090 pmd->cc->intr_handle.fd = sockfd; 1091 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 1092 pmd->cc->socket = NULL; 1093 pmd->cc->dev = dev; 1094 TAILQ_INIT(&pmd->cc->msg_queue); 1095 1096 ret = rte_intr_callback_register(&pmd->cc->intr_handle, 1097 memif_intr_handler, pmd->cc); 1098 if (ret < 0) { 1099 MIF_LOG(ERR, "Failed to register interrupt callback for control fd"); 1100 goto error; 1101 } 1102 1103 return 0; 1104 1105 error: 1106 if (sockfd >= 0) { 1107 close(sockfd); 1108 sockfd = -1; 1109 } 1110 if (pmd->cc != NULL) { 1111 rte_free(pmd->cc); 1112 pmd->cc = NULL; 1113 } 1114 return -1; 1115 } 1116