1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/ioctl.h> 11 #include <errno.h> 12 13 #include <rte_version.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <ethdev_driver.h> 17 #include <ethdev_vdev.h> 18 #include <rte_malloc.h> 19 #include <rte_kvargs.h> 20 #include <rte_bus_vdev.h> 21 #include <rte_hash.h> 22 #include <rte_jhash.h> 23 #include <rte_string_fns.h> 24 25 #include "rte_eth_memif.h" 26 #include "memif_socket.h" 27 28 static void memif_intr_handler(void *arg); 29 30 static ssize_t 31 memif_msg_send(int fd, memif_msg_t *msg, int afd) 32 { 33 struct msghdr mh = { 0 }; 34 struct iovec iov[1]; 35 struct cmsghdr *cmsg; 36 char ctl[CMSG_SPACE(sizeof(int))]; 37 38 iov[0].iov_base = msg; 39 iov[0].iov_len = sizeof(memif_msg_t); 40 mh.msg_iov = iov; 41 mh.msg_iovlen = 1; 42 43 if (afd > 0) { 44 memset(&ctl, 0, sizeof(ctl)); 45 mh.msg_control = ctl; 46 mh.msg_controllen = sizeof(ctl); 47 cmsg = CMSG_FIRSTHDR(&mh); 48 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 49 cmsg->cmsg_level = SOL_SOCKET; 50 cmsg->cmsg_type = SCM_RIGHTS; 51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 52 } 53 54 return sendmsg(fd, &mh, 0); 55 } 56 57 static int 58 memif_msg_send_from_queue(struct memif_control_channel *cc) 59 { 60 ssize_t size; 61 int ret = 0; 62 struct memif_msg_queue_elt *e; 63 64 e = TAILQ_FIRST(&cc->msg_queue); 65 if (e == NULL) 66 return 0; 67 68 size = memif_msg_send(rte_intr_fd_get(cc->intr_handle), &e->msg, 69 e->fd); 70 if (size != sizeof(memif_msg_t)) { 71 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 72 ret = -1; 73 } else { 74 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 75 } 76 TAILQ_REMOVE(&cc->msg_queue, e, next); 77 rte_free(e); 78 79 return ret; 80 } 81 82 static struct memif_msg_queue_elt * 83 memif_msg_enq(struct memif_control_channel *cc) 84 { 85 struct memif_msg_queue_elt *e; 86 87 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 88 if (e == NULL) { 89 MIF_LOG(ERR, "Failed to allocate control message."); 90 return NULL; 91 } 92 93 e->fd = -1; 94 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 95 96 return e; 97 } 98 99 void 100 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 101 int err_code) 102 { 103 struct memif_msg_queue_elt *e; 104 struct pmd_internals *pmd; 105 memif_msg_disconnect_t *d; 106 107 if (cc == NULL) { 108 MIF_LOG(DEBUG, "Missing control channel."); 109 return; 110 } 111 112 e = memif_msg_enq(cc); 113 if (e == NULL) { 114 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 115 return; 116 } 117 118 d = &e->msg.disconnect; 119 120 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 121 d->code = err_code; 122 123 if (reason != NULL) { 124 strlcpy((char *)d->string, reason, sizeof(d->string)); 125 if (cc->dev != NULL) { 126 pmd = cc->dev->data->dev_private; 127 strlcpy(pmd->local_disc_string, reason, 128 sizeof(pmd->local_disc_string)); 129 } 130 } 131 } 132 133 static int 134 memif_msg_enq_hello(struct memif_control_channel *cc) 135 { 136 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 137 memif_msg_hello_t *h; 138 139 if (e == NULL) 140 return -1; 141 142 h = &e->msg.hello; 143 144 e->msg.type = MEMIF_MSG_TYPE_HELLO; 145 h->min_version = MEMIF_VERSION; 146 h->max_version = MEMIF_VERSION; 147 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 148 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 149 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 150 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 151 152 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 153 154 return 0; 155 } 156 157 static int 158 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 159 { 160 struct pmd_internals *pmd = dev->data->dev_private; 161 memif_msg_hello_t *h = &msg->hello; 162 163 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 164 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 165 return -1; 166 } 167 168 /* Set parameters for active connection */ 169 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1, 170 pmd->cfg.num_c2s_rings); 171 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1, 172 pmd->cfg.num_s2c_rings); 173 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 174 pmd->cfg.log2_ring_size); 175 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 176 177 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 178 179 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name); 180 181 return 0; 182 } 183 184 static int 185 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 186 { 187 memif_msg_init_t *i = &msg->init; 188 struct memif_socket_dev_list_elt *elt; 189 struct pmd_internals *pmd; 190 struct rte_eth_dev *dev; 191 192 if (i->version != MEMIF_VERSION) { 193 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 194 return -1; 195 } 196 197 if (cc->socket == NULL) { 198 memif_msg_enq_disconnect(cc, "Device error", 0); 199 return -1; 200 } 201 202 /* Find device with requested ID */ 203 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 204 dev = elt->dev; 205 pmd = dev->data->dev_private; 206 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 207 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) { 208 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 209 ETH_MEMIF_FLAG_CONNECTED)) { 210 memif_msg_enq_disconnect(cc, 211 "Already connected", 0); 212 return -1; 213 } 214 215 /* assign control channel to device */ 216 cc->dev = dev; 217 pmd->cc = cc; 218 219 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 220 memif_msg_enq_disconnect(pmd->cc, 221 "Only ethernet mode supported", 222 0); 223 return -1; 224 } 225 226 strlcpy(pmd->remote_name, (char *)i->name, 227 sizeof(pmd->remote_name)); 228 229 if (*pmd->secret != '\0') { 230 if (*i->secret == '\0') { 231 memif_msg_enq_disconnect(pmd->cc, 232 "Secret required", 0); 233 return -1; 234 } 235 if (strncmp(pmd->secret, (char *)i->secret, 236 ETH_MEMIF_SECRET_SIZE) != 0) { 237 memif_msg_enq_disconnect(pmd->cc, 238 "Incorrect secret", 0); 239 return -1; 240 } 241 } 242 243 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 244 return 0; 245 } 246 } 247 248 /* ID not found on this socket */ 249 MIF_LOG(DEBUG, "ID %u not found.", i->id); 250 memif_msg_enq_disconnect(cc, "ID not found", 0); 251 return -1; 252 } 253 254 static int 255 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 256 int fd) 257 { 258 struct pmd_internals *pmd = dev->data->dev_private; 259 struct pmd_process_private *proc_private = dev->process_private; 260 memif_msg_add_region_t *ar = &msg->add_region; 261 struct memif_region *r; 262 263 if (fd < 0) { 264 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 265 return -1; 266 } 267 268 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 269 ar->index != proc_private->regions_num || 270 proc_private->regions[ar->index] != NULL) { 271 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 272 return -1; 273 } 274 275 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 276 if (r == NULL) { 277 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 278 return -ENOMEM; 279 } 280 281 r->fd = fd; 282 r->region_size = ar->size; 283 r->addr = NULL; 284 285 proc_private->regions[ar->index] = r; 286 proc_private->regions_num++; 287 288 return 0; 289 } 290 291 static int 292 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 293 { 294 struct pmd_internals *pmd = dev->data->dev_private; 295 memif_msg_add_ring_t *ar = &msg->add_ring; 296 struct memif_queue *mq; 297 298 if (fd < 0) { 299 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 300 return -1; 301 } 302 303 /* check if we have enough queues */ 304 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) { 305 if (ar->index >= pmd->cfg.num_c2s_rings) { 306 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 307 return -1; 308 } 309 pmd->run.num_c2s_rings++; 310 } else { 311 if (ar->index >= pmd->cfg.num_s2c_rings) { 312 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 313 return -1; 314 } 315 pmd->run.num_s2c_rings++; 316 } 317 318 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ? 319 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 320 321 if (rte_intr_fd_set(mq->intr_handle, fd)) 322 return -1; 323 324 mq->log2_ring_size = ar->log2_ring_size; 325 mq->region = ar->region; 326 mq->ring_offset = ar->offset; 327 328 return 0; 329 } 330 331 static int 332 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 333 { 334 struct pmd_internals *pmd = dev->data->dev_private; 335 memif_msg_connect_t *c = &msg->connect; 336 int ret; 337 338 ret = memif_connect(dev); 339 if (ret < 0) 340 return ret; 341 342 strlcpy(pmd->remote_if_name, (char *)c->if_name, 343 sizeof(pmd->remote_if_name)); 344 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 345 346 return 0; 347 } 348 349 static int 350 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 351 { 352 struct pmd_internals *pmd = dev->data->dev_private; 353 memif_msg_connected_t *c = &msg->connected; 354 int ret; 355 356 ret = memif_connect(dev); 357 if (ret < 0) 358 return ret; 359 360 strlcpy(pmd->remote_if_name, (char *)c->if_name, 361 sizeof(pmd->remote_if_name)); 362 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 363 364 return 0; 365 } 366 367 static int 368 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 369 { 370 struct pmd_internals *pmd = dev->data->dev_private; 371 memif_msg_disconnect_t *d = &msg->disconnect; 372 373 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string)); 374 strlcpy(pmd->remote_disc_string, (char *)d->string, 375 sizeof(pmd->remote_disc_string)); 376 377 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string); 378 379 memset(pmd->local_disc_string, 0, 96); 380 memif_disconnect(dev); 381 return 0; 382 } 383 384 static int 385 memif_msg_enq_ack(struct rte_eth_dev *dev) 386 { 387 struct pmd_internals *pmd = dev->data->dev_private; 388 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 389 if (e == NULL) 390 return -1; 391 392 e->msg.type = MEMIF_MSG_TYPE_ACK; 393 394 return 0; 395 } 396 397 static int 398 memif_msg_enq_init(struct rte_eth_dev *dev) 399 { 400 struct pmd_internals *pmd = dev->data->dev_private; 401 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 402 memif_msg_init_t *i = &e->msg.init; 403 404 if (e == NULL) 405 return -1; 406 407 i = &e->msg.init; 408 e->msg.type = MEMIF_MSG_TYPE_INIT; 409 i->version = MEMIF_VERSION; 410 i->id = pmd->id; 411 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 412 413 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 414 415 if (*pmd->secret != '\0') 416 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 417 418 return 0; 419 } 420 421 static int 422 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 423 { 424 struct pmd_internals *pmd = dev->data->dev_private; 425 struct pmd_process_private *proc_private = dev->process_private; 426 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 427 memif_msg_add_region_t *ar; 428 struct memif_region *mr = proc_private->regions[idx]; 429 430 if (e == NULL) 431 return -1; 432 433 ar = &e->msg.add_region; 434 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 435 e->fd = mr->fd; 436 ar->index = idx; 437 ar->size = mr->region_size; 438 439 return 0; 440 } 441 442 static int 443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 444 memif_ring_type_t type) 445 { 446 struct pmd_internals *pmd = dev->data->dev_private; 447 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 448 struct memif_queue *mq; 449 memif_msg_add_ring_t *ar; 450 451 if (e == NULL) 452 return -1; 453 454 ar = &e->msg.add_ring; 455 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] : 456 dev->data->rx_queues[idx]; 457 458 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 459 e->fd = rte_intr_fd_get(mq->intr_handle); 460 ar->index = idx; 461 ar->offset = mq->ring_offset; 462 ar->region = mq->region; 463 ar->log2_ring_size = mq->log2_ring_size; 464 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0; 465 ar->private_hdr_size = 0; 466 467 return 0; 468 } 469 470 static int 471 memif_msg_enq_connect(struct rte_eth_dev *dev) 472 { 473 struct pmd_internals *pmd = dev->data->dev_private; 474 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 475 memif_msg_connect_t *c; 476 477 if (e == NULL) 478 return -1; 479 480 c = &e->msg.connect; 481 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 482 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 483 484 return 0; 485 } 486 487 static int 488 memif_msg_enq_connected(struct rte_eth_dev *dev) 489 { 490 struct pmd_internals *pmd = dev->data->dev_private; 491 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 492 memif_msg_connected_t *c; 493 494 if (e == NULL) 495 return -1; 496 497 c = &e->msg.connected; 498 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 499 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 500 501 return 0; 502 } 503 504 static void 505 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 506 { 507 struct memif_msg_queue_elt *elt; 508 struct memif_control_channel *cc = arg; 509 510 /* close control channel fd */ 511 close(rte_intr_fd_get(intr_handle)); 512 /* clear message queue */ 513 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 514 TAILQ_REMOVE(&cc->msg_queue, elt, next); 515 rte_free(elt); 516 } 517 rte_intr_instance_free(cc->intr_handle); 518 /* free control channel */ 519 rte_free(cc); 520 } 521 522 void 523 memif_disconnect(struct rte_eth_dev *dev) 524 { 525 struct pmd_internals *pmd = dev->data->dev_private; 526 struct memif_msg_queue_elt *elt, *next; 527 struct memif_queue *mq; 528 struct rte_intr_handle *ih; 529 int i; 530 int ret; 531 532 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 533 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 534 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 535 536 rte_spinlock_lock(&pmd->cc_lock); 537 if (pmd->cc != NULL) { 538 /* Clear control message queue (except disconnect message if any). */ 539 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 540 next = TAILQ_NEXT(elt, next); 541 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 542 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 543 rte_free(elt); 544 } 545 } 546 /* send disconnect message (if there is any in queue) */ 547 memif_msg_send_from_queue(pmd->cc); 548 549 /* at this point, there should be no more messages in queue */ 550 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 551 MIF_LOG(WARNING, 552 "Unexpected message(s) in message queue."); 553 } 554 555 ih = pmd->cc->intr_handle; 556 if (rte_intr_fd_get(ih) > 0) { 557 ret = rte_intr_callback_unregister(ih, 558 memif_intr_handler, 559 pmd->cc); 560 /* 561 * If callback is active (disconnecting based on 562 * received control message). 563 */ 564 if (ret == -EAGAIN) { 565 ret = rte_intr_callback_unregister_pending(ih, 566 memif_intr_handler, 567 pmd->cc, 568 memif_intr_unregister_handler); 569 } else if (ret > 0) { 570 close(rte_intr_fd_get(ih)); 571 rte_intr_instance_free(ih); 572 rte_free(pmd->cc); 573 } 574 pmd->cc = NULL; 575 if (ret <= 0) 576 MIF_LOG(WARNING, 577 "Failed to unregister control channel callback."); 578 } 579 } 580 rte_spinlock_unlock(&pmd->cc_lock); 581 582 /* unconfig interrupts */ 583 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) { 584 if (pmd->role == MEMIF_ROLE_CLIENT) { 585 if (dev->data->tx_queues != NULL) 586 mq = dev->data->tx_queues[i]; 587 else 588 continue; 589 } else { 590 if (dev->data->rx_queues != NULL) 591 mq = dev->data->rx_queues[i]; 592 else 593 continue; 594 } 595 596 if (rte_intr_fd_get(mq->intr_handle) > 0) { 597 close(rte_intr_fd_get(mq->intr_handle)); 598 rte_intr_fd_set(mq->intr_handle, -1); 599 } 600 } 601 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) { 602 if (pmd->role == MEMIF_ROLE_SERVER) { 603 if (dev->data->tx_queues != NULL) 604 mq = dev->data->tx_queues[i]; 605 else 606 continue; 607 } else { 608 if (dev->data->rx_queues != NULL) 609 mq = dev->data->rx_queues[i]; 610 else 611 continue; 612 } 613 614 if (rte_intr_fd_get(mq->intr_handle) > 0) { 615 close(rte_intr_fd_get(mq->intr_handle)); 616 rte_intr_fd_set(mq->intr_handle, -1); 617 } 618 } 619 620 memif_free_regions(dev); 621 622 /* reset connection configuration */ 623 memset(&pmd->run, 0, sizeof(pmd->run)); 624 625 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id, 626 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client"); 627 } 628 629 static int 630 memif_msg_receive(struct memif_control_channel *cc) 631 { 632 char ctl[CMSG_SPACE(sizeof(int)) + 633 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 634 struct msghdr mh = { 0 }; 635 struct iovec iov[1]; 636 memif_msg_t msg = { 0 }; 637 ssize_t size; 638 int ret = 0; 639 struct ucred *cr __rte_unused; 640 cr = 0; 641 struct cmsghdr *cmsg; 642 int afd = -1; 643 int i; 644 struct pmd_internals *pmd; 645 struct pmd_process_private *proc_private; 646 647 iov[0].iov_base = (void *)&msg; 648 iov[0].iov_len = sizeof(memif_msg_t); 649 mh.msg_iov = iov; 650 mh.msg_iovlen = 1; 651 mh.msg_control = ctl; 652 mh.msg_controllen = sizeof(ctl); 653 654 size = recvmsg(rte_intr_fd_get(cc->intr_handle), &mh, 0); 655 if (size != sizeof(memif_msg_t)) { 656 MIF_LOG(DEBUG, "Invalid message size = %zd", size); 657 if (size > 0) 658 /* 0 means end-of-file, negative size means error, 659 * don't send further disconnect message in such cases. 660 */ 661 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 662 return -1; 663 } 664 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 665 666 cmsg = CMSG_FIRSTHDR(&mh); 667 while (cmsg) { 668 if (cmsg->cmsg_level == SOL_SOCKET) { 669 if (cmsg->cmsg_type == SCM_CREDENTIALS) 670 cr = (struct ucred *)CMSG_DATA(cmsg); 671 else if (cmsg->cmsg_type == SCM_RIGHTS) 672 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 673 } 674 cmsg = CMSG_NXTHDR(&mh, cmsg); 675 } 676 677 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 678 MIF_LOG(DEBUG, "Unexpected message."); 679 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 680 return -1; 681 } 682 683 /* get device from hash data */ 684 switch (msg.type) { 685 case MEMIF_MSG_TYPE_ACK: 686 break; 687 case MEMIF_MSG_TYPE_HELLO: 688 ret = memif_msg_receive_hello(cc->dev, &msg); 689 if (ret < 0) 690 goto exit; 691 ret = memif_init_regions_and_queues(cc->dev); 692 if (ret < 0) 693 goto exit; 694 ret = memif_msg_enq_init(cc->dev); 695 if (ret < 0) 696 goto exit; 697 pmd = cc->dev->data->dev_private; 698 proc_private = cc->dev->process_private; 699 for (i = 0; i < proc_private->regions_num; i++) { 700 ret = memif_msg_enq_add_region(cc->dev, i); 701 if (ret < 0) 702 goto exit; 703 } 704 for (i = 0; i < pmd->run.num_c2s_rings; i++) { 705 ret = memif_msg_enq_add_ring(cc->dev, i, 706 MEMIF_RING_C2S); 707 if (ret < 0) 708 goto exit; 709 } 710 for (i = 0; i < pmd->run.num_s2c_rings; i++) { 711 ret = memif_msg_enq_add_ring(cc->dev, i, 712 MEMIF_RING_S2C); 713 if (ret < 0) 714 goto exit; 715 } 716 ret = memif_msg_enq_connect(cc->dev); 717 if (ret < 0) 718 goto exit; 719 break; 720 case MEMIF_MSG_TYPE_INIT: 721 /* 722 * This cc does not have an interface asociated with it. 723 * If suitable interface is found it will be assigned here. 724 */ 725 ret = memif_msg_receive_init(cc, &msg); 726 if (ret < 0) 727 goto exit; 728 ret = memif_msg_enq_ack(cc->dev); 729 if (ret < 0) 730 goto exit; 731 break; 732 case MEMIF_MSG_TYPE_ADD_REGION: 733 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 734 if (ret < 0) 735 goto exit; 736 ret = memif_msg_enq_ack(cc->dev); 737 if (ret < 0) 738 goto exit; 739 break; 740 case MEMIF_MSG_TYPE_ADD_RING: 741 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 742 if (ret < 0) 743 goto exit; 744 ret = memif_msg_enq_ack(cc->dev); 745 if (ret < 0) 746 goto exit; 747 break; 748 case MEMIF_MSG_TYPE_CONNECT: 749 ret = memif_msg_receive_connect(cc->dev, &msg); 750 if (ret < 0) 751 goto exit; 752 ret = memif_msg_enq_connected(cc->dev); 753 if (ret < 0) 754 goto exit; 755 break; 756 case MEMIF_MSG_TYPE_CONNECTED: 757 ret = memif_msg_receive_connected(cc->dev, &msg); 758 break; 759 case MEMIF_MSG_TYPE_DISCONNECT: 760 ret = memif_msg_receive_disconnect(cc->dev, &msg); 761 if (ret < 0) 762 goto exit; 763 break; 764 default: 765 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 766 ret = -1; 767 goto exit; 768 } 769 770 exit: 771 return ret; 772 } 773 774 static void 775 memif_intr_handler(void *arg) 776 { 777 struct memif_control_channel *cc = arg; 778 int ret; 779 780 ret = memif_msg_receive(cc); 781 /* if driver failed to assign device */ 782 if (cc->dev == NULL) { 783 memif_msg_send_from_queue(cc); 784 ret = rte_intr_callback_unregister_pending(cc->intr_handle, 785 memif_intr_handler, 786 cc, 787 memif_intr_unregister_handler); 788 if (ret < 0) 789 MIF_LOG(WARNING, 790 "Failed to unregister control channel callback."); 791 return; 792 } 793 /* if memif_msg_receive failed */ 794 if (ret < 0) 795 goto disconnect; 796 797 ret = memif_msg_send_from_queue(cc); 798 if (ret < 0) 799 goto disconnect; 800 801 return; 802 803 disconnect: 804 if (cc->dev == NULL) { 805 MIF_LOG(WARNING, "eth dev not allocated"); 806 return; 807 } 808 memif_disconnect(cc->dev); 809 } 810 811 static void 812 memif_listener_handler(void *arg) 813 { 814 struct memif_socket *socket = arg; 815 int sockfd; 816 int addr_len; 817 struct sockaddr_un client; 818 struct memif_control_channel *cc; 819 int ret; 820 821 addr_len = sizeof(client); 822 sockfd = accept(rte_intr_fd_get(socket->intr_handle), 823 (struct sockaddr *)&client, (socklen_t *)&addr_len); 824 if (sockfd < 0) { 825 MIF_LOG(ERR, 826 "Failed to accept connection request on socket fd %d", 827 rte_intr_fd_get(socket->intr_handle)); 828 return; 829 } 830 831 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 832 833 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 834 if (cc == NULL) { 835 MIF_LOG(ERR, "Failed to allocate control channel."); 836 goto error; 837 } 838 839 /* Allocate interrupt instance */ 840 cc->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 841 if (cc->intr_handle == NULL) { 842 MIF_LOG(ERR, "Failed to allocate intr handle"); 843 goto error; 844 } 845 846 if (rte_intr_fd_set(cc->intr_handle, sockfd)) 847 goto error; 848 849 if (rte_intr_type_set(cc->intr_handle, RTE_INTR_HANDLE_EXT)) 850 goto error; 851 852 cc->socket = socket; 853 cc->dev = NULL; 854 TAILQ_INIT(&cc->msg_queue); 855 856 ret = rte_intr_callback_register(cc->intr_handle, memif_intr_handler, 857 cc); 858 if (ret < 0) { 859 MIF_LOG(ERR, "Failed to register control channel callback."); 860 goto error; 861 } 862 863 ret = memif_msg_enq_hello(cc); 864 if (ret < 0) { 865 MIF_LOG(ERR, "Failed to enqueue hello message."); 866 goto error; 867 } 868 ret = memif_msg_send_from_queue(cc); 869 if (ret < 0) 870 goto error; 871 872 return; 873 874 error: 875 if (sockfd >= 0) { 876 close(sockfd); 877 sockfd = -1; 878 } 879 if (cc != NULL) { 880 rte_intr_instance_free(cc->intr_handle); 881 rte_free(cc); 882 } 883 } 884 885 static struct memif_socket * 886 memif_socket_create(char *key, uint8_t listener, bool is_abstract) 887 { 888 struct memif_socket *sock; 889 struct sockaddr_un un = { 0 }; 890 uint32_t sunlen; 891 int sockfd; 892 int ret; 893 int on = 1; 894 895 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 896 if (sock == NULL) { 897 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 898 return NULL; 899 } 900 901 sock->listener = listener; 902 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE); 903 TAILQ_INIT(&sock->dev_queue); 904 905 if (listener != 0) { 906 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 907 if (sockfd < 0) 908 goto error; 909 910 un.sun_family = AF_UNIX; 911 if (is_abstract) { 912 /* abstract address */ 913 un.sun_path[0] = '\0'; 914 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1); 915 sunlen = RTE_MIN(1 + strlen(sock->filename), 916 MEMIF_SOCKET_UN_SIZE) + 917 sizeof(un) - sizeof(un.sun_path); 918 } else { 919 sunlen = sizeof(un); 920 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE); 921 } 922 923 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 924 sizeof(on)); 925 if (ret < 0) 926 goto error; 927 928 ret = bind(sockfd, (struct sockaddr *)&un, sunlen); 929 if (ret < 0) 930 goto error; 931 932 ret = listen(sockfd, 1); 933 if (ret < 0) 934 goto error; 935 936 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename); 937 938 /* Allocate interrupt instance */ 939 sock->intr_handle = 940 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 941 if (sock->intr_handle == NULL) { 942 MIF_LOG(ERR, "Failed to allocate intr handle"); 943 goto error; 944 } 945 946 if (rte_intr_fd_set(sock->intr_handle, sockfd)) 947 goto error; 948 949 if (rte_intr_type_set(sock->intr_handle, RTE_INTR_HANDLE_EXT)) 950 goto error; 951 952 ret = rte_intr_callback_register(sock->intr_handle, 953 memif_listener_handler, sock); 954 if (ret < 0) { 955 MIF_LOG(ERR, "Failed to register interrupt " 956 "callback for listener socket"); 957 return NULL; 958 } 959 } 960 961 return sock; 962 963 error: 964 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno)); 965 if (sock != NULL) { 966 rte_intr_instance_free(sock->intr_handle); 967 rte_free(sock); 968 } 969 if (sockfd >= 0) 970 close(sockfd); 971 return NULL; 972 } 973 974 static struct rte_hash * 975 memif_create_socket_hash(void) 976 { 977 struct rte_hash_parameters params = { 0 }; 978 979 params.name = MEMIF_SOCKET_HASH_NAME; 980 params.entries = 256; 981 params.key_len = MEMIF_SOCKET_UN_SIZE; 982 params.hash_func = rte_jhash; 983 params.hash_func_init_val = 0; 984 params.socket_id = SOCKET_ID_ANY; 985 return rte_hash_create(¶ms); 986 } 987 988 int 989 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 990 { 991 struct pmd_internals *pmd = dev->data->dev_private; 992 struct memif_socket *socket = NULL; 993 struct memif_socket_dev_list_elt *elt; 994 struct pmd_internals *tmp_pmd; 995 struct rte_hash *hash; 996 int ret; 997 char key[MEMIF_SOCKET_UN_SIZE]; 998 999 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1000 if (hash == NULL) { 1001 hash = memif_create_socket_hash(); 1002 if (hash == NULL) { 1003 MIF_LOG(ERR, "Failed to create memif socket hash."); 1004 return -1; 1005 } 1006 } 1007 1008 memset(key, 0, MEMIF_SOCKET_UN_SIZE); 1009 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE); 1010 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 1011 if (ret < 0) { 1012 socket = memif_socket_create(key, 1013 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1, 1014 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT); 1015 if (socket == NULL) 1016 return -1; 1017 ret = rte_hash_add_key_data(hash, key, socket); 1018 if (ret < 0) { 1019 MIF_LOG(ERR, "Failed to add socket to socket hash."); 1020 return ret; 1021 } 1022 } 1023 pmd->socket_filename = socket->filename; 1024 1025 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 1026 tmp_pmd = elt->dev->data->dev_private; 1027 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) { 1028 MIF_LOG(ERR, "Two interfaces with the same id (%d) can " 1029 "not have the same role.", pmd->id); 1030 return -1; 1031 } 1032 } 1033 1034 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 1035 if (elt == NULL) { 1036 MIF_LOG(ERR, "Failed to add device to socket device list."); 1037 return -1; 1038 } 1039 elt->dev = dev; 1040 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 1041 1042 return 0; 1043 } 1044 1045 void 1046 memif_socket_remove_device(struct rte_eth_dev *dev) 1047 { 1048 struct pmd_internals *pmd = dev->data->dev_private; 1049 struct memif_socket *socket = NULL; 1050 struct memif_socket_dev_list_elt *elt, *next; 1051 struct rte_hash *hash; 1052 int ret; 1053 1054 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1055 if (hash == NULL) 1056 return; 1057 1058 if (pmd->socket_filename == NULL) 1059 return; 1060 1061 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1062 return; 1063 1064 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1065 next = TAILQ_NEXT(elt, next); 1066 if (elt->dev == dev) { 1067 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1068 rte_free(elt); 1069 pmd->socket_filename = NULL; 1070 } 1071 } 1072 1073 /* remove socket, if this was the last device using it */ 1074 if (TAILQ_EMPTY(&socket->dev_queue)) { 1075 rte_hash_del_key(hash, socket->filename); 1076 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) { 1077 /* remove listener socket file, 1078 * so we can create new one later. 1079 */ 1080 ret = remove(socket->filename); 1081 if (ret < 0) 1082 MIF_LOG(ERR, "Failed to remove socket file: %s", 1083 socket->filename); 1084 } 1085 if (pmd->role != MEMIF_ROLE_CLIENT) 1086 rte_intr_instance_free(socket->intr_handle); 1087 rte_free(socket); 1088 } 1089 } 1090 1091 int 1092 memif_connect_server(struct rte_eth_dev *dev) 1093 { 1094 struct pmd_internals *pmd = dev->data->dev_private; 1095 1096 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1097 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1098 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1099 return 0; 1100 } 1101 1102 int 1103 memif_connect_client(struct rte_eth_dev *dev) 1104 { 1105 int sockfd; 1106 int ret; 1107 uint32_t sunlen; 1108 struct sockaddr_un sun = { 0 }; 1109 struct pmd_internals *pmd = dev->data->dev_private; 1110 1111 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1112 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1113 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1114 1115 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1116 if (sockfd < 0) { 1117 MIF_LOG(ERR, "Failed to open socket."); 1118 return -1; 1119 } 1120 1121 sun.sun_family = AF_UNIX; 1122 sunlen = sizeof(struct sockaddr_un); 1123 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) { 1124 /* abstract address */ 1125 sun.sun_path[0] = '\0'; 1126 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1); 1127 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1, 1128 MEMIF_SOCKET_UN_SIZE) + 1129 sizeof(sun) - sizeof(sun.sun_path); 1130 } else { 1131 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE); 1132 } 1133 1134 ret = connect(sockfd, (struct sockaddr *)&sun, sunlen); 1135 if (ret < 0) { 1136 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename); 1137 goto error; 1138 } 1139 1140 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename); 1141 1142 pmd->cc = rte_zmalloc("memif-cc", 1143 sizeof(struct memif_control_channel), 0); 1144 if (pmd->cc == NULL) { 1145 MIF_LOG(ERR, "Failed to allocate control channel."); 1146 goto error; 1147 } 1148 1149 /* Allocate interrupt instance */ 1150 pmd->cc->intr_handle = 1151 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); 1152 if (pmd->cc->intr_handle == NULL) { 1153 MIF_LOG(ERR, "Failed to allocate intr handle"); 1154 goto error; 1155 } 1156 1157 if (rte_intr_fd_set(pmd->cc->intr_handle, sockfd)) 1158 goto error; 1159 1160 if (rte_intr_type_set(pmd->cc->intr_handle, RTE_INTR_HANDLE_EXT)) 1161 goto error; 1162 1163 pmd->cc->socket = NULL; 1164 pmd->cc->dev = dev; 1165 TAILQ_INIT(&pmd->cc->msg_queue); 1166 1167 ret = rte_intr_callback_register(pmd->cc->intr_handle, 1168 memif_intr_handler, pmd->cc); 1169 if (ret < 0) { 1170 MIF_LOG(ERR, "Failed to register interrupt callback for control fd"); 1171 goto error; 1172 } 1173 1174 return 0; 1175 1176 error: 1177 if (sockfd >= 0) { 1178 close(sockfd); 1179 sockfd = -1; 1180 } 1181 if (pmd->cc != NULL) { 1182 rte_intr_instance_free(pmd->cc->intr_handle); 1183 rte_free(pmd->cc); 1184 pmd->cc = NULL; 1185 } 1186 return -1; 1187 } 1188