1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved. 3 */ 4 5 #include <stdlib.h> 6 #include <fcntl.h> 7 #include <unistd.h> 8 #include <sys/types.h> 9 #include <sys/socket.h> 10 #include <sys/ioctl.h> 11 #include <errno.h> 12 13 #include <rte_version.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <ethdev_driver.h> 17 #include <ethdev_vdev.h> 18 #include <rte_malloc.h> 19 #include <rte_kvargs.h> 20 #include <rte_bus_vdev.h> 21 #include <rte_hash.h> 22 #include <rte_jhash.h> 23 #include <rte_string_fns.h> 24 25 #include "rte_eth_memif.h" 26 #include "memif_socket.h" 27 28 static void memif_intr_handler(void *arg); 29 30 static ssize_t 31 memif_msg_send(int fd, memif_msg_t *msg, int afd) 32 { 33 struct msghdr mh = { 0 }; 34 struct iovec iov[1]; 35 struct cmsghdr *cmsg; 36 char ctl[CMSG_SPACE(sizeof(int))]; 37 38 iov[0].iov_base = msg; 39 iov[0].iov_len = sizeof(memif_msg_t); 40 mh.msg_iov = iov; 41 mh.msg_iovlen = 1; 42 43 if (afd > 0) { 44 memset(&ctl, 0, sizeof(ctl)); 45 mh.msg_control = ctl; 46 mh.msg_controllen = sizeof(ctl); 47 cmsg = CMSG_FIRSTHDR(&mh); 48 cmsg->cmsg_len = CMSG_LEN(sizeof(int)); 49 cmsg->cmsg_level = SOL_SOCKET; 50 cmsg->cmsg_type = SCM_RIGHTS; 51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int)); 52 } 53 54 return sendmsg(fd, &mh, 0); 55 } 56 57 static int 58 memif_msg_send_from_queue(struct memif_control_channel *cc) 59 { 60 ssize_t size; 61 int ret = 0; 62 struct memif_msg_queue_elt *e; 63 64 e = TAILQ_FIRST(&cc->msg_queue); 65 if (e == NULL) 66 return 0; 67 68 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd); 69 if (size != sizeof(memif_msg_t)) { 70 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno)); 71 ret = -1; 72 } else { 73 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type); 74 } 75 TAILQ_REMOVE(&cc->msg_queue, e, next); 76 rte_free(e); 77 78 return ret; 79 } 80 81 static struct memif_msg_queue_elt * 82 memif_msg_enq(struct memif_control_channel *cc) 83 { 84 struct memif_msg_queue_elt *e; 85 86 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0); 87 if (e == NULL) { 88 MIF_LOG(ERR, "Failed to allocate control message."); 89 return NULL; 90 } 91 92 e->fd = -1; 93 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next); 94 95 return e; 96 } 97 98 void 99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason, 100 int err_code) 101 { 102 struct memif_msg_queue_elt *e; 103 struct pmd_internals *pmd; 104 memif_msg_disconnect_t *d; 105 106 if (cc == NULL) { 107 MIF_LOG(DEBUG, "Missing control channel."); 108 return; 109 } 110 111 e = memif_msg_enq(cc); 112 if (e == NULL) { 113 MIF_LOG(WARNING, "Failed to enqueue disconnect message."); 114 return; 115 } 116 117 d = &e->msg.disconnect; 118 119 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT; 120 d->code = err_code; 121 122 if (reason != NULL) { 123 strlcpy((char *)d->string, reason, sizeof(d->string)); 124 if (cc->dev != NULL) { 125 pmd = cc->dev->data->dev_private; 126 strlcpy(pmd->local_disc_string, reason, 127 sizeof(pmd->local_disc_string)); 128 } 129 } 130 } 131 132 static int 133 memif_msg_enq_hello(struct memif_control_channel *cc) 134 { 135 struct memif_msg_queue_elt *e = memif_msg_enq(cc); 136 memif_msg_hello_t *h; 137 138 if (e == NULL) 139 return -1; 140 141 h = &e->msg.hello; 142 143 e->msg.type = MEMIF_MSG_TYPE_HELLO; 144 h->min_version = MEMIF_VERSION; 145 h->max_version = MEMIF_VERSION; 146 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 147 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS; 148 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1; 149 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE; 150 151 strlcpy((char *)h->name, rte_version(), sizeof(h->name)); 152 153 return 0; 154 } 155 156 static int 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg) 158 { 159 struct pmd_internals *pmd = dev->data->dev_private; 160 memif_msg_hello_t *h = &msg->hello; 161 162 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) { 163 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0); 164 return -1; 165 } 166 167 /* Set parameters for active connection */ 168 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1, 169 pmd->cfg.num_c2s_rings); 170 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1, 171 pmd->cfg.num_s2c_rings); 172 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size, 173 pmd->cfg.log2_ring_size); 174 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size; 175 176 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name)); 177 178 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name); 179 180 return 0; 181 } 182 183 static int 184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg) 185 { 186 memif_msg_init_t *i = &msg->init; 187 struct memif_socket_dev_list_elt *elt; 188 struct pmd_internals *pmd; 189 struct rte_eth_dev *dev; 190 191 if (i->version != MEMIF_VERSION) { 192 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0); 193 return -1; 194 } 195 196 if (cc->socket == NULL) { 197 memif_msg_enq_disconnect(cc, "Device error", 0); 198 return -1; 199 } 200 201 /* Find device with requested ID */ 202 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) { 203 dev = elt->dev; 204 pmd = dev->data->dev_private; 205 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) && 206 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) { 207 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING | 208 ETH_MEMIF_FLAG_CONNECTED)) { 209 memif_msg_enq_disconnect(cc, 210 "Already connected", 0); 211 return -1; 212 } 213 214 /* assign control channel to device */ 215 cc->dev = dev; 216 pmd->cc = cc; 217 218 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) { 219 memif_msg_enq_disconnect(pmd->cc, 220 "Only ethernet mode supported", 221 0); 222 return -1; 223 } 224 225 strlcpy(pmd->remote_name, (char *)i->name, 226 sizeof(pmd->remote_name)); 227 228 if (*pmd->secret != '\0') { 229 if (*i->secret == '\0') { 230 memif_msg_enq_disconnect(pmd->cc, 231 "Secret required", 0); 232 return -1; 233 } 234 if (strncmp(pmd->secret, (char *)i->secret, 235 ETH_MEMIF_SECRET_SIZE) != 0) { 236 memif_msg_enq_disconnect(pmd->cc, 237 "Incorrect secret", 0); 238 return -1; 239 } 240 } 241 242 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING; 243 return 0; 244 } 245 } 246 247 /* ID not found on this socket */ 248 MIF_LOG(DEBUG, "ID %u not found.", i->id); 249 memif_msg_enq_disconnect(cc, "ID not found", 0); 250 return -1; 251 } 252 253 static int 254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg, 255 int fd) 256 { 257 struct pmd_internals *pmd = dev->data->dev_private; 258 struct pmd_process_private *proc_private = dev->process_private; 259 memif_msg_add_region_t *ar = &msg->add_region; 260 struct memif_region *r; 261 262 if (fd < 0) { 263 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0); 264 return -1; 265 } 266 267 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || 268 ar->index != proc_private->regions_num || 269 proc_private->regions[ar->index] != NULL) { 270 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0); 271 return -1; 272 } 273 274 r = rte_zmalloc("region", sizeof(struct memif_region), 0); 275 if (r == NULL) { 276 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0); 277 return -ENOMEM; 278 } 279 280 r->fd = fd; 281 r->region_size = ar->size; 282 r->addr = NULL; 283 284 proc_private->regions[ar->index] = r; 285 proc_private->regions_num++; 286 287 return 0; 288 } 289 290 static int 291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd) 292 { 293 struct pmd_internals *pmd = dev->data->dev_private; 294 memif_msg_add_ring_t *ar = &msg->add_ring; 295 struct memif_queue *mq; 296 297 if (fd < 0) { 298 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0); 299 return -1; 300 } 301 302 /* check if we have enough queues */ 303 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) { 304 if (ar->index >= pmd->cfg.num_c2s_rings) { 305 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 306 return -1; 307 } 308 pmd->run.num_c2s_rings++; 309 } else { 310 if (ar->index >= pmd->cfg.num_s2c_rings) { 311 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0); 312 return -1; 313 } 314 pmd->run.num_s2c_rings++; 315 } 316 317 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ? 318 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index]; 319 320 mq->intr_handle.fd = fd; 321 mq->log2_ring_size = ar->log2_ring_size; 322 mq->region = ar->region; 323 mq->ring_offset = ar->offset; 324 325 return 0; 326 } 327 328 static int 329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg) 330 { 331 struct pmd_internals *pmd = dev->data->dev_private; 332 memif_msg_connect_t *c = &msg->connect; 333 int ret; 334 335 ret = memif_connect(dev); 336 if (ret < 0) 337 return ret; 338 339 strlcpy(pmd->remote_if_name, (char *)c->if_name, 340 sizeof(pmd->remote_if_name)); 341 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 342 343 return 0; 344 } 345 346 static int 347 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg) 348 { 349 struct pmd_internals *pmd = dev->data->dev_private; 350 memif_msg_connected_t *c = &msg->connected; 351 int ret; 352 353 ret = memif_connect(dev); 354 if (ret < 0) 355 return ret; 356 357 strlcpy(pmd->remote_if_name, (char *)c->if_name, 358 sizeof(pmd->remote_if_name)); 359 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name); 360 361 return 0; 362 } 363 364 static int 365 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg) 366 { 367 struct pmd_internals *pmd = dev->data->dev_private; 368 memif_msg_disconnect_t *d = &msg->disconnect; 369 370 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string)); 371 strlcpy(pmd->remote_disc_string, (char *)d->string, 372 sizeof(pmd->remote_disc_string)); 373 374 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string); 375 376 memset(pmd->local_disc_string, 0, 96); 377 memif_disconnect(dev); 378 return 0; 379 } 380 381 static int 382 memif_msg_enq_ack(struct rte_eth_dev *dev) 383 { 384 struct pmd_internals *pmd = dev->data->dev_private; 385 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 386 if (e == NULL) 387 return -1; 388 389 e->msg.type = MEMIF_MSG_TYPE_ACK; 390 391 return 0; 392 } 393 394 static int 395 memif_msg_enq_init(struct rte_eth_dev *dev) 396 { 397 struct pmd_internals *pmd = dev->data->dev_private; 398 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 399 memif_msg_init_t *i = &e->msg.init; 400 401 if (e == NULL) 402 return -1; 403 404 i = &e->msg.init; 405 e->msg.type = MEMIF_MSG_TYPE_INIT; 406 i->version = MEMIF_VERSION; 407 i->id = pmd->id; 408 i->mode = MEMIF_INTERFACE_MODE_ETHERNET; 409 410 strlcpy((char *)i->name, rte_version(), sizeof(i->name)); 411 412 if (*pmd->secret != '\0') 413 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret)); 414 415 return 0; 416 } 417 418 static int 419 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx) 420 { 421 struct pmd_internals *pmd = dev->data->dev_private; 422 struct pmd_process_private *proc_private = dev->process_private; 423 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 424 memif_msg_add_region_t *ar; 425 struct memif_region *mr = proc_private->regions[idx]; 426 427 if (e == NULL) 428 return -1; 429 430 ar = &e->msg.add_region; 431 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION; 432 e->fd = mr->fd; 433 ar->index = idx; 434 ar->size = mr->region_size; 435 436 return 0; 437 } 438 439 static int 440 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx, 441 memif_ring_type_t type) 442 { 443 struct pmd_internals *pmd = dev->data->dev_private; 444 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 445 struct memif_queue *mq; 446 memif_msg_add_ring_t *ar; 447 448 if (e == NULL) 449 return -1; 450 451 ar = &e->msg.add_ring; 452 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] : 453 dev->data->rx_queues[idx]; 454 455 e->msg.type = MEMIF_MSG_TYPE_ADD_RING; 456 e->fd = mq->intr_handle.fd; 457 ar->index = idx; 458 ar->offset = mq->ring_offset; 459 ar->region = mq->region; 460 ar->log2_ring_size = mq->log2_ring_size; 461 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0; 462 ar->private_hdr_size = 0; 463 464 return 0; 465 } 466 467 static int 468 memif_msg_enq_connect(struct rte_eth_dev *dev) 469 { 470 struct pmd_internals *pmd = dev->data->dev_private; 471 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 472 memif_msg_connect_t *c; 473 474 if (e == NULL) 475 return -1; 476 477 c = &e->msg.connect; 478 e->msg.type = MEMIF_MSG_TYPE_CONNECT; 479 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 480 481 return 0; 482 } 483 484 static int 485 memif_msg_enq_connected(struct rte_eth_dev *dev) 486 { 487 struct pmd_internals *pmd = dev->data->dev_private; 488 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc); 489 memif_msg_connected_t *c; 490 491 if (e == NULL) 492 return -1; 493 494 c = &e->msg.connected; 495 e->msg.type = MEMIF_MSG_TYPE_CONNECTED; 496 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name)); 497 498 return 0; 499 } 500 501 static void 502 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg) 503 { 504 struct memif_msg_queue_elt *elt; 505 struct memif_control_channel *cc = arg; 506 507 /* close control channel fd */ 508 close(intr_handle->fd); 509 /* clear message queue */ 510 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) { 511 TAILQ_REMOVE(&cc->msg_queue, elt, next); 512 rte_free(elt); 513 } 514 /* free control channel */ 515 rte_free(cc); 516 } 517 518 void 519 memif_disconnect(struct rte_eth_dev *dev) 520 { 521 struct pmd_internals *pmd = dev->data->dev_private; 522 struct memif_msg_queue_elt *elt, *next; 523 struct memif_queue *mq; 524 struct rte_intr_handle *ih; 525 int i; 526 int ret; 527 528 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 529 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; 530 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED; 531 532 rte_spinlock_lock(&pmd->cc_lock); 533 if (pmd->cc != NULL) { 534 /* Clear control message queue (except disconnect message if any). */ 535 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) { 536 next = TAILQ_NEXT(elt, next); 537 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) { 538 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next); 539 rte_free(elt); 540 } 541 } 542 /* send disconnect message (if there is any in queue) */ 543 memif_msg_send_from_queue(pmd->cc); 544 545 /* at this point, there should be no more messages in queue */ 546 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) { 547 MIF_LOG(WARNING, 548 "Unexpected message(s) in message queue."); 549 } 550 551 ih = &pmd->cc->intr_handle; 552 if (ih->fd > 0) { 553 ret = rte_intr_callback_unregister(ih, 554 memif_intr_handler, 555 pmd->cc); 556 /* 557 * If callback is active (disconnecting based on 558 * received control message). 559 */ 560 if (ret == -EAGAIN) { 561 ret = rte_intr_callback_unregister_pending(ih, 562 memif_intr_handler, 563 pmd->cc, 564 memif_intr_unregister_handler); 565 } else if (ret > 0) { 566 close(ih->fd); 567 rte_free(pmd->cc); 568 } 569 pmd->cc = NULL; 570 if (ret <= 0) 571 MIF_LOG(WARNING, 572 "Failed to unregister control channel callback."); 573 } 574 } 575 rte_spinlock_unlock(&pmd->cc_lock); 576 577 /* unconfig interrupts */ 578 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) { 579 if (pmd->role == MEMIF_ROLE_CLIENT) { 580 if (dev->data->tx_queues != NULL) 581 mq = dev->data->tx_queues[i]; 582 else 583 continue; 584 } else { 585 if (dev->data->rx_queues != NULL) 586 mq = dev->data->rx_queues[i]; 587 else 588 continue; 589 } 590 if (mq->intr_handle.fd > 0) { 591 close(mq->intr_handle.fd); 592 mq->intr_handle.fd = -1; 593 } 594 } 595 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) { 596 if (pmd->role == MEMIF_ROLE_SERVER) { 597 if (dev->data->tx_queues != NULL) 598 mq = dev->data->tx_queues[i]; 599 else 600 continue; 601 } else { 602 if (dev->data->rx_queues != NULL) 603 mq = dev->data->rx_queues[i]; 604 else 605 continue; 606 } 607 if (mq->intr_handle.fd > 0) { 608 close(mq->intr_handle.fd); 609 mq->intr_handle.fd = -1; 610 } 611 } 612 613 memif_free_regions(dev); 614 615 /* reset connection configuration */ 616 memset(&pmd->run, 0, sizeof(pmd->run)); 617 618 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id, 619 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client"); 620 } 621 622 static int 623 memif_msg_receive(struct memif_control_channel *cc) 624 { 625 char ctl[CMSG_SPACE(sizeof(int)) + 626 CMSG_SPACE(sizeof(struct ucred))] = { 0 }; 627 struct msghdr mh = { 0 }; 628 struct iovec iov[1]; 629 memif_msg_t msg = { 0 }; 630 ssize_t size; 631 int ret = 0; 632 struct ucred *cr __rte_unused; 633 cr = 0; 634 struct cmsghdr *cmsg; 635 int afd = -1; 636 int i; 637 struct pmd_internals *pmd; 638 struct pmd_process_private *proc_private; 639 640 iov[0].iov_base = (void *)&msg; 641 iov[0].iov_len = sizeof(memif_msg_t); 642 mh.msg_iov = iov; 643 mh.msg_iovlen = 1; 644 mh.msg_control = ctl; 645 mh.msg_controllen = sizeof(ctl); 646 647 size = recvmsg(cc->intr_handle.fd, &mh, 0); 648 if (size != sizeof(memif_msg_t)) { 649 MIF_LOG(DEBUG, "Invalid message size = %zd", size); 650 if (size > 0) 651 /* 0 means end-of-file, negative size means error, 652 * don't send further disconnect message in such cases. 653 */ 654 memif_msg_enq_disconnect(cc, "Invalid message size", 0); 655 return -1; 656 } 657 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type); 658 659 cmsg = CMSG_FIRSTHDR(&mh); 660 while (cmsg) { 661 if (cmsg->cmsg_level == SOL_SOCKET) { 662 if (cmsg->cmsg_type == SCM_CREDENTIALS) 663 cr = (struct ucred *)CMSG_DATA(cmsg); 664 else if (cmsg->cmsg_type == SCM_RIGHTS) 665 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int)); 666 } 667 cmsg = CMSG_NXTHDR(&mh, cmsg); 668 } 669 670 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) { 671 MIF_LOG(DEBUG, "Unexpected message."); 672 memif_msg_enq_disconnect(cc, "Unexpected message", 0); 673 return -1; 674 } 675 676 /* get device from hash data */ 677 switch (msg.type) { 678 case MEMIF_MSG_TYPE_ACK: 679 break; 680 case MEMIF_MSG_TYPE_HELLO: 681 ret = memif_msg_receive_hello(cc->dev, &msg); 682 if (ret < 0) 683 goto exit; 684 ret = memif_init_regions_and_queues(cc->dev); 685 if (ret < 0) 686 goto exit; 687 ret = memif_msg_enq_init(cc->dev); 688 if (ret < 0) 689 goto exit; 690 pmd = cc->dev->data->dev_private; 691 proc_private = cc->dev->process_private; 692 for (i = 0; i < proc_private->regions_num; i++) { 693 ret = memif_msg_enq_add_region(cc->dev, i); 694 if (ret < 0) 695 goto exit; 696 } 697 for (i = 0; i < pmd->run.num_c2s_rings; i++) { 698 ret = memif_msg_enq_add_ring(cc->dev, i, 699 MEMIF_RING_C2S); 700 if (ret < 0) 701 goto exit; 702 } 703 for (i = 0; i < pmd->run.num_s2c_rings; i++) { 704 ret = memif_msg_enq_add_ring(cc->dev, i, 705 MEMIF_RING_S2C); 706 if (ret < 0) 707 goto exit; 708 } 709 ret = memif_msg_enq_connect(cc->dev); 710 if (ret < 0) 711 goto exit; 712 break; 713 case MEMIF_MSG_TYPE_INIT: 714 /* 715 * This cc does not have an interface asociated with it. 716 * If suitable interface is found it will be assigned here. 717 */ 718 ret = memif_msg_receive_init(cc, &msg); 719 if (ret < 0) 720 goto exit; 721 ret = memif_msg_enq_ack(cc->dev); 722 if (ret < 0) 723 goto exit; 724 break; 725 case MEMIF_MSG_TYPE_ADD_REGION: 726 ret = memif_msg_receive_add_region(cc->dev, &msg, afd); 727 if (ret < 0) 728 goto exit; 729 ret = memif_msg_enq_ack(cc->dev); 730 if (ret < 0) 731 goto exit; 732 break; 733 case MEMIF_MSG_TYPE_ADD_RING: 734 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd); 735 if (ret < 0) 736 goto exit; 737 ret = memif_msg_enq_ack(cc->dev); 738 if (ret < 0) 739 goto exit; 740 break; 741 case MEMIF_MSG_TYPE_CONNECT: 742 ret = memif_msg_receive_connect(cc->dev, &msg); 743 if (ret < 0) 744 goto exit; 745 ret = memif_msg_enq_connected(cc->dev); 746 if (ret < 0) 747 goto exit; 748 break; 749 case MEMIF_MSG_TYPE_CONNECTED: 750 ret = memif_msg_receive_connected(cc->dev, &msg); 751 break; 752 case MEMIF_MSG_TYPE_DISCONNECT: 753 ret = memif_msg_receive_disconnect(cc->dev, &msg); 754 if (ret < 0) 755 goto exit; 756 break; 757 default: 758 memif_msg_enq_disconnect(cc, "Unknown message type", 0); 759 ret = -1; 760 goto exit; 761 } 762 763 exit: 764 return ret; 765 } 766 767 static void 768 memif_intr_handler(void *arg) 769 { 770 struct memif_control_channel *cc = arg; 771 int ret; 772 773 ret = memif_msg_receive(cc); 774 /* if driver failed to assign device */ 775 if (cc->dev == NULL) { 776 memif_msg_send_from_queue(cc); 777 ret = rte_intr_callback_unregister_pending(&cc->intr_handle, 778 memif_intr_handler, 779 cc, 780 memif_intr_unregister_handler); 781 if (ret < 0) 782 MIF_LOG(WARNING, 783 "Failed to unregister control channel callback."); 784 return; 785 } 786 /* if memif_msg_receive failed */ 787 if (ret < 0) 788 goto disconnect; 789 790 ret = memif_msg_send_from_queue(cc); 791 if (ret < 0) 792 goto disconnect; 793 794 return; 795 796 disconnect: 797 if (cc->dev == NULL) { 798 MIF_LOG(WARNING, "eth dev not allocated"); 799 return; 800 } 801 memif_disconnect(cc->dev); 802 } 803 804 static void 805 memif_listener_handler(void *arg) 806 { 807 struct memif_socket *socket = arg; 808 int sockfd; 809 int addr_len; 810 struct sockaddr_un client; 811 struct memif_control_channel *cc; 812 int ret; 813 814 addr_len = sizeof(client); 815 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client, 816 (socklen_t *)&addr_len); 817 if (sockfd < 0) { 818 MIF_LOG(ERR, 819 "Failed to accept connection request on socket fd %d", 820 socket->intr_handle.fd); 821 return; 822 } 823 824 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename); 825 826 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0); 827 if (cc == NULL) { 828 MIF_LOG(ERR, "Failed to allocate control channel."); 829 goto error; 830 } 831 832 cc->intr_handle.fd = sockfd; 833 cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 834 cc->socket = socket; 835 cc->dev = NULL; 836 TAILQ_INIT(&cc->msg_queue); 837 838 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc); 839 if (ret < 0) { 840 MIF_LOG(ERR, "Failed to register control channel callback."); 841 goto error; 842 } 843 844 ret = memif_msg_enq_hello(cc); 845 if (ret < 0) { 846 MIF_LOG(ERR, "Failed to enqueue hello message."); 847 goto error; 848 } 849 ret = memif_msg_send_from_queue(cc); 850 if (ret < 0) 851 goto error; 852 853 return; 854 855 error: 856 if (sockfd >= 0) { 857 close(sockfd); 858 sockfd = -1; 859 } 860 if (cc != NULL) 861 rte_free(cc); 862 } 863 864 static struct memif_socket * 865 memif_socket_create(char *key, uint8_t listener, bool is_abstract) 866 { 867 struct memif_socket *sock; 868 struct sockaddr_un un = { 0 }; 869 uint32_t sunlen; 870 int sockfd; 871 int ret; 872 int on = 1; 873 874 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0); 875 if (sock == NULL) { 876 MIF_LOG(ERR, "Failed to allocate memory for memif socket"); 877 return NULL; 878 } 879 880 sock->listener = listener; 881 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE); 882 TAILQ_INIT(&sock->dev_queue); 883 884 if (listener != 0) { 885 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 886 if (sockfd < 0) 887 goto error; 888 889 un.sun_family = AF_UNIX; 890 if (is_abstract) { 891 /* abstract address */ 892 un.sun_path[0] = '\0'; 893 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1); 894 sunlen = RTE_MIN(1 + strlen(sock->filename), 895 MEMIF_SOCKET_UN_SIZE) + 896 sizeof(un) - sizeof(un.sun_path); 897 } else { 898 sunlen = sizeof(un); 899 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE); 900 } 901 902 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on, 903 sizeof(on)); 904 if (ret < 0) 905 goto error; 906 907 ret = bind(sockfd, (struct sockaddr *)&un, sunlen); 908 if (ret < 0) 909 goto error; 910 911 ret = listen(sockfd, 1); 912 if (ret < 0) 913 goto error; 914 915 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename); 916 917 sock->intr_handle.fd = sockfd; 918 sock->intr_handle.type = RTE_INTR_HANDLE_EXT; 919 ret = rte_intr_callback_register(&sock->intr_handle, 920 memif_listener_handler, sock); 921 if (ret < 0) { 922 MIF_LOG(ERR, "Failed to register interrupt " 923 "callback for listener socket"); 924 return NULL; 925 } 926 } 927 928 return sock; 929 930 error: 931 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno)); 932 if (sock != NULL) 933 rte_free(sock); 934 if (sockfd >= 0) 935 close(sockfd); 936 return NULL; 937 } 938 939 static struct rte_hash * 940 memif_create_socket_hash(void) 941 { 942 struct rte_hash_parameters params = { 0 }; 943 944 params.name = MEMIF_SOCKET_HASH_NAME; 945 params.entries = 256; 946 params.key_len = MEMIF_SOCKET_UN_SIZE; 947 params.hash_func = rte_jhash; 948 params.hash_func_init_val = 0; 949 params.socket_id = SOCKET_ID_ANY; 950 return rte_hash_create(¶ms); 951 } 952 953 int 954 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename) 955 { 956 struct pmd_internals *pmd = dev->data->dev_private; 957 struct memif_socket *socket = NULL; 958 struct memif_socket_dev_list_elt *elt; 959 struct pmd_internals *tmp_pmd; 960 struct rte_hash *hash; 961 int ret; 962 char key[MEMIF_SOCKET_UN_SIZE]; 963 964 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 965 if (hash == NULL) { 966 hash = memif_create_socket_hash(); 967 if (hash == NULL) { 968 MIF_LOG(ERR, "Failed to create memif socket hash."); 969 return -1; 970 } 971 } 972 973 memset(key, 0, MEMIF_SOCKET_UN_SIZE); 974 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE); 975 ret = rte_hash_lookup_data(hash, key, (void **)&socket); 976 if (ret < 0) { 977 socket = memif_socket_create(key, 978 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1, 979 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT); 980 if (socket == NULL) 981 return -1; 982 ret = rte_hash_add_key_data(hash, key, socket); 983 if (ret < 0) { 984 MIF_LOG(ERR, "Failed to add socket to socket hash."); 985 return ret; 986 } 987 } 988 pmd->socket_filename = socket->filename; 989 990 TAILQ_FOREACH(elt, &socket->dev_queue, next) { 991 tmp_pmd = elt->dev->data->dev_private; 992 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) { 993 MIF_LOG(ERR, "Two interfaces with the same id (%d) can " 994 "not have the same role.", pmd->id); 995 return -1; 996 } 997 } 998 999 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0); 1000 if (elt == NULL) { 1001 MIF_LOG(ERR, "Failed to add device to socket device list."); 1002 return -1; 1003 } 1004 elt->dev = dev; 1005 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next); 1006 1007 return 0; 1008 } 1009 1010 void 1011 memif_socket_remove_device(struct rte_eth_dev *dev) 1012 { 1013 struct pmd_internals *pmd = dev->data->dev_private; 1014 struct memif_socket *socket = NULL; 1015 struct memif_socket_dev_list_elt *elt, *next; 1016 struct rte_hash *hash; 1017 int ret; 1018 1019 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME); 1020 if (hash == NULL) 1021 return; 1022 1023 if (pmd->socket_filename == NULL) 1024 return; 1025 1026 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0) 1027 return; 1028 1029 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) { 1030 next = TAILQ_NEXT(elt, next); 1031 if (elt->dev == dev) { 1032 TAILQ_REMOVE(&socket->dev_queue, elt, next); 1033 rte_free(elt); 1034 pmd->socket_filename = NULL; 1035 } 1036 } 1037 1038 /* remove socket, if this was the last device using it */ 1039 if (TAILQ_EMPTY(&socket->dev_queue)) { 1040 rte_hash_del_key(hash, socket->filename); 1041 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) { 1042 /* remove listener socket file, 1043 * so we can create new one later. 1044 */ 1045 ret = remove(socket->filename); 1046 if (ret < 0) 1047 MIF_LOG(ERR, "Failed to remove socket file: %s", 1048 socket->filename); 1049 } 1050 rte_free(socket); 1051 } 1052 } 1053 1054 int 1055 memif_connect_server(struct rte_eth_dev *dev) 1056 { 1057 struct pmd_internals *pmd = dev->data->dev_private; 1058 1059 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1060 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1061 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1062 return 0; 1063 } 1064 1065 int 1066 memif_connect_client(struct rte_eth_dev *dev) 1067 { 1068 int sockfd; 1069 int ret; 1070 uint32_t sunlen; 1071 struct sockaddr_un sun = { 0 }; 1072 struct pmd_internals *pmd = dev->data->dev_private; 1073 1074 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1075 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE); 1076 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED; 1077 1078 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0); 1079 if (sockfd < 0) { 1080 MIF_LOG(ERR, "Failed to open socket."); 1081 return -1; 1082 } 1083 1084 sun.sun_family = AF_UNIX; 1085 sunlen = sizeof(struct sockaddr_un); 1086 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) { 1087 /* abstract address */ 1088 sun.sun_path[0] = '\0'; 1089 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1); 1090 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1, 1091 MEMIF_SOCKET_UN_SIZE) + 1092 sizeof(sun) - sizeof(sun.sun_path); 1093 } else { 1094 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE); 1095 } 1096 1097 ret = connect(sockfd, (struct sockaddr *)&sun, sunlen); 1098 if (ret < 0) { 1099 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename); 1100 goto error; 1101 } 1102 1103 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename); 1104 1105 pmd->cc = rte_zmalloc("memif-cc", 1106 sizeof(struct memif_control_channel), 0); 1107 if (pmd->cc == NULL) { 1108 MIF_LOG(ERR, "Failed to allocate control channel."); 1109 goto error; 1110 } 1111 1112 pmd->cc->intr_handle.fd = sockfd; 1113 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT; 1114 pmd->cc->socket = NULL; 1115 pmd->cc->dev = dev; 1116 TAILQ_INIT(&pmd->cc->msg_queue); 1117 1118 ret = rte_intr_callback_register(&pmd->cc->intr_handle, 1119 memif_intr_handler, pmd->cc); 1120 if (ret < 0) { 1121 MIF_LOG(ERR, "Failed to register interrupt callback for control fd"); 1122 goto error; 1123 } 1124 1125 return 0; 1126 1127 error: 1128 if (sockfd >= 0) { 1129 close(sockfd); 1130 sockfd = -1; 1131 } 1132 if (pmd->cc != NULL) { 1133 rte_free(pmd->cc); 1134 pmd->cc = NULL; 1135 } 1136 return -1; 1137 } 1138