1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/ioctl.h> 11 #include <errno.h> 12 13 #include <rte_version.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <ethdev_driver.h> 17 #include <ethdev_vdev.h> 18 #include <rte_malloc.h> 19 #include <rte_kvargs.h> 20 #include <rte_bus_vdev.h> 21 #include <rte_hash.h> 22 #include <rte_jhash.h> 23 #include <rte_string_fns.h> 24 25 #include "rte_eth_memif.h" 26 #include "memif_socket.h" 27 28 static void memif_intr_handler(void *arg); 29 30 static ssize_t 31 memif_msg_send(int fd, memif_msg_t *msg, int afd) 32 { 33 struct msghdr mh = { 0 }; 34 struct iovec iov[1]; 35 struct cmsghdr *cmsg; 36 char ctl[CMSG_SPACE(sizeof(int))]; 37 38 iov[0].iov_base = msg; 39 iov[0].iov_len = sizeof(memif_msg_t); 40 mh.msg_iov = iov; 41 mh.msg_iovlen = 1; 42 43 if (afd > 0) { 44 memset(&ctl, 0, sizeof(ctl)); 45 mh.msg_control = ctl; 46 mh.msg_controllen = sizeof(ctl); 47 cmsg = CMSG_FIRSTHDR(&mh); 48 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 49 cmsg->cmsg_level = SOL_SOCKET; 50 cmsg->cmsg_type = SCM_RIGHTS; 51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 52 } 53 54 return sendmsg(fd, &mh, 0); 55 } 56 57 static int 58 memif_msg_send_from_queue(struct memif_control_channel *cc) 59 { 60 ssize_t size; 61 int ret = 0; 62 struct memif_msg_queue_elt *e; 63 64 e = TAILQ_FIRST(&cc->msg_queue); 65 if (e == NULL) 66 return 0; 67 68 if (rte_intr_fd_get(cc->intr_handle) < 0) 69 return -1; 70 71 size = memif_msg_send(rte_intr_fd_get(cc->intr_handle), &e->msg, 72 e->fd); 73 if (size != sizeof(memif_msg_t)) { 74 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 75 ret = -1; 76 } else { 77 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 78 } 79 TAILQ_REMOVE(&cc->msg_queue, e, next); 80 rte_free(e); 81 82 return ret; 83 } 84 85 static struct memif_msg_queue_elt * 86 memif_msg_enq(struct memif_control_channel *cc) 87 { 88 struct memif_msg_queue_elt *e; 89 90 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 91 if (e == NULL) { 92 MIF_LOG(ERR, "Failed to allocate control message."); 93 return NULL; 94 } 95 96 e->fd = -1; 97 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 98 99 return e; 100 } 101 102 void 103 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 104 int err_code) 105 { 106 struct memif_msg_queue_elt *e; 107 struct pmd_internals *pmd; 108 memif_msg_disconnect_t *d; 109 110 if (cc == NULL) { 111 MIF_LOG(DEBUG, "Missing control channel."); 112 return; 113 } 114 115 e = memif_msg_enq(cc); 116 if (e == NULL) { 117 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 118 return; 119 } 120 121 d = &e->msg.disconnect; 122 123 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 124 d->code = err_code; 125 126 if (reason != NULL) { 127 strlcpy((char *)d->string, reason, sizeof(d->string)); 128 if (cc->dev != NULL) { 129 pmd = cc->dev->data->dev_private; 130 strlcpy(pmd->local_disc_string, reason, 131 sizeof(pmd->local_disc_string)); 132 } 133 } 134 } 135 136 static int 137 memif_msg_enq_hello(struct memif_control_channel *cc) 138 { 139 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 140 memif_msg_hello_t *h; 141 142 if (e == NULL) 143 return -1; 144 145 h = &e->msg.hello; 146 147 e->msg.type = MEMIF_MSG_TYPE_HELLO; 148 h->min_version = MEMIF_VERSION; 149 h->max_version = MEMIF_VERSION; 150 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 151 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 152 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 153 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 154 155 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 156 157 return 0; 158 } 159 160 static int 161 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 162 { 163 struct pmd_internals *pmd = dev->data->dev_private; 164 memif_msg_hello_t *h = &msg->hello; 165 166 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 167 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 168 return -1; 169 } 170 171 /* Set parameters for active connection */ 172 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1, 173 pmd->cfg.num_c2s_rings); 174 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1, 175 pmd->cfg.num_s2c_rings); 176 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 177 pmd->cfg.log2_ring_size); 178 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 179 180 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 181 182 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name); 183 184 return 0; 185 } 186 187 static int 188 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 189 { 190 memif_msg_init_t *i = &msg->init; 191 struct memif_socket_dev_list_elt *elt; 192 struct pmd_internals *pmd; 193 struct rte_eth_dev *dev; 194 195 if (i->version != MEMIF_VERSION) { 196 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 197 return -1; 198 } 199 200 if (cc->socket == NULL) { 201 memif_msg_enq_disconnect(cc, "Device error", 0); 202 return -1; 203 } 204 205 /* Find device with requested ID */ 206 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 207 dev = elt->dev; 208 pmd = dev->data->dev_private; 209 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 210 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) { 211 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 212 ETH_MEMIF_FLAG_CONNECTED)) { 213 memif_msg_enq_disconnect(cc, 214 "Already connected", 0); 215 return -1; 216 } 217 218 /* assign control channel to device */ 219 cc->dev = dev; 220 pmd->cc = cc; 221 222 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 223 memif_msg_enq_disconnect(pmd->cc, 224 "Only ethernet mode supported", 225 0); 226 return -1; 227 } 228 229 strlcpy(pmd->remote_name, (char *)i->name, 230 sizeof(pmd->remote_name)); 231 232 if (*pmd->secret != '\0') { 233 if (*i->secret == '\0') { 234 memif_msg_enq_disconnect(pmd->cc, 235 "Secret required", 0); 236 return -1; 237 } 238 if (strncmp(pmd->secret, (char *)i->secret, 239 ETH_MEMIF_SECRET_SIZE) != 0) { 240 memif_msg_enq_disconnect(pmd->cc, 241 "Incorrect secret", 0); 242 return -1; 243 } 244 } 245 246 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 247 return 0; 248 } 249 } 250 251 /* ID not found on this socket */ 252 MIF_LOG(DEBUG, "ID %u not found.", i->id); 253 memif_msg_enq_disconnect(cc, "ID not found", 0); 254 return -1; 255 } 256 257 static int 258 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 259 int fd) 260 { 261 struct pmd_internals *pmd = dev->data->dev_private; 262 struct pmd_process_private *proc_private = dev->process_private; 263 memif_msg_add_region_t *ar = &msg->add_region; 264 struct memif_region *r; 265 266 if (fd < 0) { 267 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 268 return -1; 269 } 270 271 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 272 ar->index != proc_private->regions_num || 273 proc_private->regions[ar->index] != NULL) { 274 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 275 return -1; 276 } 277 278 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 279 if (r == NULL) { 280 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 281 return -ENOMEM; 282 } 283 284 r->fd = fd; 285 r->region_size = ar->size; 286 r->addr = NULL; 287 288 proc_private->regions[ar->index] = r; 289 proc_private->regions_num++; 290 291 return 0; 292 } 293 294 static int 295 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 296 { 297 struct pmd_internals *pmd = dev->data->dev_private; 298 memif_msg_add_ring_t *ar = &msg->add_ring; 299 struct memif_queue *mq; 300 301 if (fd < 0) { 302 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 303 return -1; 304 } 305 306 /* check if we have enough queues */ 307 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) { 308 if (ar->index >= pmd->cfg.num_c2s_rings) { 309 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 310 return -1; 311 } 312 pmd->run.num_c2s_rings++; 313 } else { 314 if (ar->index >= pmd->cfg.num_s2c_rings) { 315 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 316 return -1; 317 } 318 pmd->run.num_s2c_rings++; 319 } 320 321 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ? 322 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 323 324 if (rte_intr_fd_set(mq->intr_handle, fd)) 325 return -1; 326 327 mq->log2_ring_size = ar->log2_ring_size; 328 mq->region = ar->region; 329 mq->ring_offset = ar->offset; 330 331 return 0; 332 } 333 334 static int 335 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 336 { 337 struct pmd_internals *pmd = dev->data->dev_private; 338 memif_msg_connect_t *c = &msg->connect; 339 int ret; 340 341 ret = memif_connect(dev); 342 if (ret < 0) 343 return ret; 344 345 strlcpy(pmd->remote_if_name, (char *)c->if_name, 346 sizeof(pmd->remote_if_name)); 347 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 348 349 return 0; 350 } 351 352 static int 353 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 354 { 355 struct pmd_internals *pmd = dev->data->dev_private; 356 memif_msg_connected_t *c = &msg->connected; 357 int ret; 358 359 ret = memif_connect(dev); 360 if (ret < 0) 361 return ret; 362 363 strlcpy(pmd->remote_if_name, (char *)c->if_name, 364 sizeof(pmd->remote_if_name)); 365 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 366 367 return 0; 368 } 369 370 static int 371 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 372 { 373 struct pmd_internals *pmd = dev->data->dev_private; 374 memif_msg_disconnect_t *d = &msg->disconnect; 375 376 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string)); 377 strlcpy(pmd->remote_disc_string, (char *)d->string, 378 sizeof(pmd->remote_disc_string)); 379 380 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string); 381 382 memset(pmd->local_disc_string, 0, 96); 383 memif_disconnect(dev); 384 return 0; 385 } 386 387 static int 388 memif_msg_enq_ack(struct rte_eth_dev *dev) 389 { 390 struct pmd_internals *pmd = dev->data->dev_private; 391 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 392 if (e == NULL) 393 return -1; 394 395 e->msg.type = MEMIF_MSG_TYPE_ACK; 396 397 return 0; 398 } 399 400 static int 401 memif_msg_enq_init(struct rte_eth_dev *dev) 402 { 403 struct pmd_internals *pmd = dev->data->dev_private; 404 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 405 memif_msg_init_t *i; 406 407 if (e == NULL) 408 return -1; 409 i = &e->msg.init; 410 e->msg.type = MEMIF_MSG_TYPE_INIT; 411 i->version = MEMIF_VERSION; 412 i->id = pmd->id; 413 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 414 415 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 416 417 if (*pmd->secret != '\0') 418 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 419 420 return 0; 421 } 422 423 static int 424 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 425 { 426 struct pmd_internals *pmd = dev->data->dev_private; 427 struct pmd_process_private *proc_private = dev->process_private; 428 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 429 memif_msg_add_region_t *ar; 430 struct memif_region *mr = proc_private->regions[idx]; 431 432 if (e == NULL) 433 return -1; 434 435 ar = &e->msg.add_region; 436 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 437 e->fd = mr->fd; 438 ar->index = idx; 439 ar->size = mr->region_size; 440 441 return 0; 442 } 443 444 static int 445 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 446 memif_ring_type_t type) 447 { 448 struct pmd_internals *pmd = dev->data->dev_private; 449 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 450 struct memif_queue *mq; 451 memif_msg_add_ring_t *ar; 452 453 if (e == NULL) 454 return -1; 455 456 ar = &e->msg.add_ring; 457 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] : 458 dev->data->rx_queues[idx]; 459 460 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 461 e->fd = rte_intr_fd_get(mq->intr_handle); 462 ar->index = idx; 463 ar->offset = mq->ring_offset; 464 ar->region = mq->region; 465 ar->log2_ring_size = mq->log2_ring_size; 466 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0; 467 ar->private_hdr_size = 0; 468 469 return 0; 470 } 471 472 static int 473 memif_msg_enq_connect(struct rte_eth_dev *dev) 474 { 475 struct pmd_internals *pmd = dev->data->dev_private; 476 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 477 memif_msg_connect_t *c; 478 479 if (e == NULL) 480 return -1; 481 482 c = &e->msg.connect; 483 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 484 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 485 486 return 0; 487 } 488 489 static int 490 memif_msg_enq_connected(struct rte_eth_dev *dev) 491 { 492 struct pmd_internals *pmd = dev->data->dev_private; 493 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 494 memif_msg_connected_t *c; 495 496 if (e == NULL) 497 return -1; 498 499 c = &e->msg.connected; 500 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 501 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 502 503 return 0; 504 } 505 506 static void 507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 508 { 509 struct memif_msg_queue_elt *elt; 510 struct memif_control_channel *cc = arg; 511 512 /* close control channel fd */ 513 if (rte_intr_fd_get(intr_handle) >= 0) 514 close(rte_intr_fd_get(intr_handle)); 515 /* clear message queue */ 516 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 517 TAILQ_REMOVE(&cc->msg_queue, elt, next); 518 rte_free(elt); 519 } 520 rte_intr_instance_free(cc->intr_handle); 521 /* free control channel */ 522 rte_free(cc); 523 } 524 525 void 526 memif_disconnect(struct rte_eth_dev *dev) 527 { 528 struct pmd_internals *pmd = dev->data->dev_private; 529 struct memif_msg_queue_elt *elt, *next; 530 struct memif_queue *mq; 531 struct rte_intr_handle *ih; 532 int i; 533 int ret; 534 535 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 536 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 537 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 538 539 rte_spinlock_lock(&pmd->cc_lock); 540 if (pmd->cc != NULL) { 541 /* Clear control message queue (except disconnect message if any). */ 542 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 543 next = TAILQ_NEXT(elt, next); 544 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 545 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 546 rte_free(elt); 547 } 548 } 549 /* send disconnect message (if there is any in queue) */ 550 memif_msg_send_from_queue(pmd->cc); 551 552 /* at this point, there should be no more messages in queue */ 553 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 554 MIF_LOG(WARNING, 555 "Unexpected message(s) in message queue."); 556 } 557 558 ih = pmd->cc->intr_handle; 559 if (rte_intr_fd_get(ih) > 0) { 560 ret = rte_intr_callback_unregister(ih, 561 memif_intr_handler, 562 pmd->cc); 563 /* 564 * If callback is active (disconnecting based on 565 * received control message). 566 */ 567 if (ret == -EAGAIN) { 568 ret = rte_intr_callback_unregister_pending(ih, 569 memif_intr_handler, 570 pmd->cc, 571 memif_intr_unregister_handler); 572 } else if (ret > 0) { 573 close(rte_intr_fd_get(ih)); 574 rte_intr_instance_free(ih); 575 rte_free(pmd->cc); 576 } 577 pmd->cc = NULL; 578 if (ret <= 0) 579 MIF_LOG(WARNING, 580 "Failed to unregister control channel callback."); 581 } 582 } 583 rte_spinlock_unlock(&pmd->cc_lock); 584 585 /* unconfig interrupts */ 586 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) { 587 if (pmd->role == MEMIF_ROLE_CLIENT) { 588 if (dev->data->tx_queues != NULL) 589 mq = dev->data->tx_queues[i]; 590 else 591 continue; 592 } else { 593 if (dev->data->rx_queues != NULL) 594 mq = dev->data->rx_queues[i]; 595 else 596 continue; 597 } 598 599 if (rte_intr_fd_get(mq->intr_handle) > 0) { 600 close(rte_intr_fd_get(mq->intr_handle)); 601 rte_intr_fd_set(mq->intr_handle, -1); 602 } 603 } 604 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) { 605 if (pmd->role == MEMIF_ROLE_SERVER) { 606 if (dev->data->tx_queues != NULL) 607 mq = dev->data->tx_queues[i]; 608 else 609 continue; 610 } else { 611 if (dev->data->rx_queues != NULL) 612 mq = dev->data->rx_queues[i]; 613 else 614 continue; 615 } 616 617 if (rte_intr_fd_get(mq->intr_handle) > 0) { 618 close(rte_intr_fd_get(mq->intr_handle)); 619 rte_intr_fd_set(mq->intr_handle, -1); 620 } 621 } 622 623 memif_free_regions(dev); 624 625 /* reset connection configuration */ 626 memset(&pmd->run, 0, sizeof(pmd->run)); 627 628 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id, 629 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client"); 630 } 631 632 static int 633 memif_msg_receive(struct memif_control_channel *cc) 634 { 635 char ctl[CMSG_SPACE(sizeof(int)) + 636 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 637 struct msghdr mh = { 0 }; 638 struct iovec iov[1]; 639 memif_msg_t msg = { 0 }; 640 ssize_t size; 641 int ret = 0; 642 struct ucred *cr __rte_unused; 643 cr = 0; 644 struct cmsghdr *cmsg; 645 int afd = -1; 646 int i; 647 struct pmd_internals *pmd; 648 struct pmd_process_private *proc_private; 649 650 iov[0].iov_base = (void *)&msg; 651 iov[0].iov_len = sizeof(memif_msg_t); 652 mh.msg_iov = iov; 653 mh.msg_iovlen = 1; 654 mh.msg_control = ctl; 655 mh.msg_controllen = sizeof(ctl); 656 657 if (rte_intr_fd_get(cc->intr_handle) < 0) 658 return -1; 659 660 size = recvmsg(rte_intr_fd_get(cc->intr_handle), &mh, 0); 661 if (size != sizeof(memif_msg_t)) { 662 MIF_LOG(DEBUG, "Invalid message size = %zd", size); 663 if (size > 0) 664 /* 0 means end-of-file, negative size means error, 665 * don't send further disconnect message in such cases. 666 */ 667 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 668 return -1; 669 } 670 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 671 672 cmsg = CMSG_FIRSTHDR(&mh); 673 while (cmsg) { 674 if (cmsg->cmsg_level == SOL_SOCKET) { 675 if (cmsg->cmsg_type == SCM_CREDENTIALS) 676 cr = (struct ucred *)CMSG_DATA(cmsg); 677 else if (cmsg->cmsg_type == SCM_RIGHTS) 678 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 679 } 680 cmsg = CMSG_NXTHDR(&mh, cmsg); 681 } 682 683 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 684 MIF_LOG(DEBUG, "Unexpected message."); 685 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 686 return -1; 687 } 688 689 /* get device from hash data */ 690 switch (msg.type) { 691 case MEMIF_MSG_TYPE_ACK: 692 break; 693 case MEMIF_MSG_TYPE_HELLO: 694 ret = memif_msg_receive_hello(cc->dev, &msg); 695 if (ret < 0) 696 goto exit; 697 ret = memif_init_regions_and_queues(cc->dev); 698 if (ret < 0) 699 goto exit; 700 ret = memif_msg_enq_init(cc->dev); 701 if (ret < 0) 702 goto exit; 703 pmd = cc->dev->data->dev_private; 704 proc_private = cc->dev->process_private; 705 for (i = 0; i < proc_private->regions_num; i++) { 706 ret = memif_msg_enq_add_region(cc->dev, i); 707 if (ret < 0) 708 goto exit; 709 } 710 for (i = 0; i < pmd->run.num_c2s_rings; i++) { 711 ret = memif_msg_enq_add_ring(cc->dev, i, 712 MEMIF_RING_C2S); 713 if (ret < 0) 714 goto exit; 715 } 716 for (i = 0; i < pmd->run.num_s2c_rings; i++) { 717 ret = memif_msg_enq_add_ring(cc->dev, i, 718 MEMIF_RING_S2C); 719 if (ret < 0) 720 goto exit; 721 } 722 ret = memif_msg_enq_connect(cc->dev); 723 if (ret < 0) 724 goto exit; 725 break; 726 case MEMIF_MSG_TYPE_INIT: 727 /* 728 * This cc does not have an interface associated with it. 729 * If suitable interface is found it will be assigned here. 730 */ 731 ret = memif_msg_receive_init(cc, &msg); 732 if (ret < 0) 733 goto exit; 734 ret = memif_msg_enq_ack(cc->dev); 735 if (ret < 0) 736 goto exit; 737 break; 738 case MEMIF_MSG_TYPE_ADD_REGION: 739 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 740 if (ret < 0) 741 goto exit; 742 ret = memif_msg_enq_ack(cc->dev); 743 if (ret < 0) 744 goto exit; 745 break; 746 case MEMIF_MSG_TYPE_ADD_RING: 747 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 748 if (ret < 0) 749 goto exit; 750 ret = memif_msg_enq_ack(cc->dev); 751 if (ret < 0) 752 goto exit; 753 break; 754 case MEMIF_MSG_TYPE_CONNECT: 755 ret = memif_msg_receive_connect(cc->dev, &msg); 756 if (ret < 0) 757 goto exit; 758 ret = memif_msg_enq_connected(cc->dev); 759 if (ret < 0) 760 goto exit; 761 break; 762 case MEMIF_MSG_TYPE_CONNECTED: 763 ret = memif_msg_receive_connected(cc->dev, &msg); 764 break; 765 case MEMIF_MSG_TYPE_DISCONNECT: 766 ret = memif_msg_receive_disconnect(cc->dev, &msg); 767 if (ret < 0) 768 goto exit; 769 break; 770 default: 771 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 772 ret = -1; 773 goto exit; 774 } 775 776 exit: 777 return ret; 778 } 779 780 static void 781 memif_intr_handler(void *arg) 782 { 783 struct memif_control_channel *cc = arg; 784 int ret; 785 786 ret = memif_msg_receive(cc); 787 /* if driver failed to assign device */ 788 if (cc->dev == NULL) { 789 memif_msg_send_from_queue(cc); 790 ret = rte_intr_callback_unregister_pending(cc->intr_handle, 791 memif_intr_handler, 792 cc, 793 memif_intr_unregister_handler); 794 if (ret < 0) 795 MIF_LOG(WARNING, 796 "Failed to unregister control channel callback."); 797 return; 798 } 799 /* if memif_msg_receive failed */ 800 if (ret < 0) 801 goto disconnect; 802 803 ret = memif_msg_send_from_queue(cc); 804 if (ret < 0) 805 goto disconnect; 806 807 return; 808 809 disconnect: 810 if (cc->dev == NULL) { 811 MIF_LOG(WARNING, "eth dev not allocated"); 812 return; 813 } 814 memif_disconnect(cc->dev); 815 } 816 817 static void 818 memif_listener_handler(void *arg) 819 { 820 struct memif_socket *socket = arg; 821 int sockfd; 822 int addr_len; 823 struct sockaddr_un client; 824 struct memif_control_channel *cc; 825 int ret; 826 827 addr_len = sizeof(client); 828 sockfd = accept(rte_intr_fd_get(socket->intr_handle), 829 (struct sockaddr *)&client, (socklen_t *)&addr_len); 830 if (sockfd < 0) { 831 MIF_LOG(ERR, 832 "Failed to accept connection request on socket fd %d", 833 rte_intr_fd_get(socket->intr_handle)); 834 return; 835 } 836 837 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 838 839 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 840 if (cc == NULL) { 841 MIF_LOG(ERR, "Failed to allocate control channel."); 842 goto error; 843 } 844 845 /* Allocate interrupt instance */ 846 cc->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 847 if (cc->intr_handle == NULL) { 848 MIF_LOG(ERR, "Failed to allocate intr handle"); 849 goto error; 850 } 851 852 if (rte_intr_fd_set(cc->intr_handle, sockfd)) 853 goto error; 854 855 if (rte_intr_type_set(cc->intr_handle, RTE_INTR_HANDLE_EXT)) 856 goto error; 857 858 cc->socket = socket; 859 cc->dev = NULL; 860 TAILQ_INIT(&cc->msg_queue); 861 862 ret = rte_intr_callback_register(cc->intr_handle, memif_intr_handler, 863 cc); 864 if (ret < 0) { 865 MIF_LOG(ERR, "Failed to register control channel callback."); 866 goto error; 867 } 868 869 ret = memif_msg_enq_hello(cc); 870 if (ret < 0) { 871 MIF_LOG(ERR, "Failed to enqueue hello message."); 872 goto error; 873 } 874 ret = memif_msg_send_from_queue(cc); 875 if (ret < 0) 876 goto error; 877 878 return; 879 880 error: 881 if (sockfd >= 0) { 882 close(sockfd); 883 sockfd = -1; 884 } 885 if (cc != NULL) { 886 rte_intr_instance_free(cc->intr_handle); 887 rte_free(cc); 888 } 889 } 890 891 static struct memif_socket * 892 memif_socket_create(char *key, uint8_t listener, bool is_abstract) 893 { 894 struct memif_socket *sock; 895 struct sockaddr_un un = { 0 }; 896 uint32_t sunlen; 897 int sockfd; 898 int ret; 899 int on = 1; 900 901 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 902 if (sock == NULL) { 903 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 904 return NULL; 905 } 906 907 sock->listener = listener; 908 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE); 909 TAILQ_INIT(&sock->dev_queue); 910 911 if (listener != 0) { 912 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 913 if (sockfd < 0) 914 goto error; 915 916 un.sun_family = AF_UNIX; 917 if (is_abstract) { 918 /* abstract address */ 919 un.sun_path[0] = '\0'; 920 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1); 921 sunlen = RTE_MIN(1 + strlen(sock->filename), 922 MEMIF_SOCKET_UN_SIZE) + 923 sizeof(un) - sizeof(un.sun_path); 924 } else { 925 sunlen = sizeof(un); 926 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE); 927 } 928 929 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 930 sizeof(on)); 931 if (ret < 0) 932 goto error; 933 934 ret = bind(sockfd, (struct sockaddr *)&un, sunlen); 935 if (ret < 0) 936 goto error; 937 938 ret = listen(sockfd, 1); 939 if (ret < 0) 940 goto error; 941 942 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename); 943 944 /* Allocate interrupt instance */ 945 sock->intr_handle = 946 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 947 if (sock->intr_handle == NULL) { 948 MIF_LOG(ERR, "Failed to allocate intr handle"); 949 goto error; 950 } 951 952 if (rte_intr_fd_set(sock->intr_handle, sockfd)) 953 goto error; 954 955 if (rte_intr_type_set(sock->intr_handle, RTE_INTR_HANDLE_EXT)) 956 goto error; 957 958 ret = rte_intr_callback_register(sock->intr_handle, 959 memif_listener_handler, sock); 960 if (ret < 0) { 961 MIF_LOG(ERR, "Failed to register interrupt " 962 "callback for listener socket"); 963 return NULL; 964 } 965 } 966 967 return sock; 968 969 error: 970 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno)); 971 if (sock != NULL) { 972 rte_intr_instance_free(sock->intr_handle); 973 rte_free(sock); 974 } 975 if (sockfd >= 0) 976 close(sockfd); 977 return NULL; 978 } 979 980 static struct rte_hash * 981 memif_create_socket_hash(void) 982 { 983 struct rte_hash_parameters params = { 0 }; 984 985 params.name = MEMIF_SOCKET_HASH_NAME; 986 params.entries = 256; 987 params.key_len = MEMIF_SOCKET_UN_SIZE; 988 params.hash_func = rte_jhash; 989 params.hash_func_init_val = 0; 990 params.socket_id = SOCKET_ID_ANY; 991 return rte_hash_create(¶ms); 992 } 993 994 int 995 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 996 { 997 struct pmd_internals *pmd = dev->data->dev_private; 998 struct memif_socket *socket = NULL; 999 struct memif_socket_dev_list_elt *elt; 1000 struct pmd_internals *tmp_pmd; 1001 struct rte_hash *hash; 1002 int ret; 1003 char key[MEMIF_SOCKET_UN_SIZE]; 1004 1005 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1006 if (hash == NULL) { 1007 hash = memif_create_socket_hash(); 1008 if (hash == NULL) { 1009 MIF_LOG(ERR, "Failed to create memif socket hash."); 1010 return -1; 1011 } 1012 } 1013 1014 memset(key, 0, MEMIF_SOCKET_UN_SIZE); 1015 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE); 1016 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 1017 if (ret < 0) { 1018 socket = memif_socket_create(key, 1019 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1, 1020 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT); 1021 if (socket == NULL) 1022 return -1; 1023 ret = rte_hash_add_key_data(hash, key, socket); 1024 if (ret < 0) { 1025 MIF_LOG(ERR, "Failed to add socket to socket hash."); 1026 return ret; 1027 } 1028 } 1029 pmd->socket_filename = socket->filename; 1030 1031 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 1032 tmp_pmd = elt->dev->data->dev_private; 1033 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) { 1034 MIF_LOG(ERR, "Two interfaces with the same id (%d) can " 1035 "not have the same role.", pmd->id); 1036 return -1; 1037 } 1038 } 1039 1040 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 1041 if (elt == NULL) { 1042 MIF_LOG(ERR, "Failed to add device to socket device list."); 1043 return -1; 1044 } 1045 elt->dev = dev; 1046 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 1047 1048 return 0; 1049 } 1050 1051 void 1052 memif_socket_remove_device(struct rte_eth_dev *dev) 1053 { 1054 struct pmd_internals *pmd = dev->data->dev_private; 1055 struct memif_socket *socket = NULL; 1056 struct memif_socket_dev_list_elt *elt, *next; 1057 struct rte_hash *hash; 1058 int ret; 1059 1060 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1061 if (hash == NULL) 1062 return; 1063 1064 if (pmd->socket_filename == NULL) 1065 return; 1066 1067 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1068 return; 1069 1070 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1071 next = TAILQ_NEXT(elt, next); 1072 if (elt->dev == dev) { 1073 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1074 rte_free(elt); 1075 pmd->socket_filename = NULL; 1076 } 1077 } 1078 1079 /* remove socket, if this was the last device using it */ 1080 if (TAILQ_EMPTY(&socket->dev_queue)) { 1081 rte_hash_del_key(hash, socket->filename); 1082 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) { 1083 /* remove listener socket file, 1084 * so we can create new one later. 1085 */ 1086 ret = remove(socket->filename); 1087 if (ret < 0) 1088 MIF_LOG(ERR, "Failed to remove socket file: %s", 1089 socket->filename); 1090 } 1091 if (pmd->role != MEMIF_ROLE_CLIENT) 1092 rte_intr_instance_free(socket->intr_handle); 1093 rte_free(socket); 1094 } 1095 } 1096 1097 int 1098 memif_connect_server(struct rte_eth_dev *dev) 1099 { 1100 struct pmd_internals *pmd = dev->data->dev_private; 1101 1102 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1103 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1104 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1105 return 0; 1106 } 1107 1108 int 1109 memif_connect_client(struct rte_eth_dev *dev) 1110 { 1111 int sockfd; 1112 int ret; 1113 uint32_t sunlen; 1114 struct sockaddr_un sun = { 0 }; 1115 struct pmd_internals *pmd = dev->data->dev_private; 1116 1117 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1118 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1119 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1120 1121 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1122 if (sockfd < 0) { 1123 MIF_LOG(ERR, "Failed to open socket."); 1124 return -1; 1125 } 1126 1127 sun.sun_family = AF_UNIX; 1128 sunlen = sizeof(struct sockaddr_un); 1129 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) { 1130 /* abstract address */ 1131 sun.sun_path[0] = '\0'; 1132 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1); 1133 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1, 1134 MEMIF_SOCKET_UN_SIZE) + 1135 sizeof(sun) - sizeof(sun.sun_path); 1136 } else { 1137 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE); 1138 } 1139 1140 ret = connect(sockfd, (struct sockaddr *)&sun, sunlen); 1141 if (ret < 0) { 1142 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename); 1143 goto error; 1144 } 1145 1146 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename); 1147 1148 pmd->cc = rte_zmalloc("memif-cc", 1149 sizeof(struct memif_control_channel), 0); 1150 if (pmd->cc == NULL) { 1151 MIF_LOG(ERR, "Failed to allocate control channel."); 1152 goto error; 1153 } 1154 1155 /* Allocate interrupt instance */ 1156 pmd->cc->intr_handle = 1157 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 1158 if (pmd->cc->intr_handle == NULL) { 1159 MIF_LOG(ERR, "Failed to allocate intr handle"); 1160 goto error; 1161 } 1162 1163 if (rte_intr_fd_set(pmd->cc->intr_handle, sockfd)) 1164 goto error; 1165 1166 if (rte_intr_type_set(pmd->cc->intr_handle, RTE_INTR_HANDLE_EXT)) 1167 goto error; 1168 1169 pmd->cc->socket = NULL; 1170 pmd->cc->dev = dev; 1171 TAILQ_INIT(&pmd->cc->msg_queue); 1172 1173 ret = rte_intr_callback_register(pmd->cc->intr_handle, 1174 memif_intr_handler, pmd->cc); 1175 if (ret < 0) { 1176 MIF_LOG(ERR, "Failed to register interrupt callback for control fd"); 1177 goto error; 1178 } 1179 1180 return 0; 1181 1182 error: 1183 if (sockfd >= 0) { 1184 close(sockfd); 1185 sockfd = -1; 1186 } 1187 if (pmd->cc != NULL) { 1188 rte_intr_instance_free(pmd->cc->intr_handle); 1189 rte_free(pmd->cc); 1190 pmd->cc = NULL; 1191 } 1192 return -1; 1193 } 1194