1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <sys/un.h> 8 #include <fcntl.h> 9 #include <unistd.h> 10 #include <inttypes.h> 11 #include <dirent.h> 12 #include <errno.h> 13 14 #include <sys/queue.h> 15 #include <sys/types.h> 16 #include <sys/stat.h> 17 #include <sys/socket.h> 18 #include <sys/select.h> 19 20 #include <rte_string_fns.h> 21 #include <rte_malloc.h> 22 #include <rte_memory.h> 23 #include <rte_mempool.h> 24 #include <rte_log.h> 25 #include <rte_atomic.h> 26 #include <rte_spinlock.h> 27 28 #include <libvirt/libvirt.h> 29 30 #include "channel_manager.h" 31 #include "channel_commands.h" 32 #include "channel_monitor.h" 33 #include "power_manager.h" 34 35 36 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1 37 38 /* Global pointer to libvirt connection */ 39 static virConnectPtr global_vir_conn_ptr; 40 41 static unsigned char *global_cpumaps; 42 static virVcpuInfo *global_vircpuinfo; 43 static size_t global_maplen; 44 45 static unsigned int global_n_host_cpus; 46 static bool global_hypervisor_available; 47 48 /* 49 * Represents a single Virtual Machine 50 */ 51 struct virtual_machine_info { 52 char name[CHANNEL_MGR_MAX_NAME_LEN]; 53 uint16_t pcpu_map[RTE_MAX_LCORE]; 54 struct channel_info *channels[RTE_MAX_LCORE]; 55 char channel_mask[RTE_MAX_LCORE]; 56 uint8_t num_channels; 57 enum vm_status status; 58 virDomainPtr domainPtr; 59 virDomainInfo info; 60 rte_spinlock_t config_spinlock; 61 LIST_ENTRY(virtual_machine_info) vms_info; 62 }; 63 64 LIST_HEAD(, virtual_machine_info) vm_list_head; 65 66 static struct virtual_machine_info * 67 find_domain_by_name(const char *name) 68 { 69 struct virtual_machine_info *info; 70 LIST_FOREACH(info, &vm_list_head, vms_info) { 71 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1)) 72 return info; 73 } 74 return NULL; 75 } 76 77 static int 78 update_pcpus_mask(struct virtual_machine_info *vm_info) 79 { 80 virVcpuInfoPtr cpuinfo; 81 unsigned i, j; 82 int n_vcpus; 83 84 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 85 86 if (!virDomainIsActive(vm_info->domainPtr)) { 87 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr, 88 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen, 89 VIR_DOMAIN_AFFECT_CONFIG); 90 if (n_vcpus < 0) { 91 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 92 "in-active VM '%s'\n", vm_info->name); 93 return -1; 94 } 95 goto update_pcpus; 96 } 97 98 memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)* 99 RTE_MAX_LCORE); 100 101 cpuinfo = global_vircpuinfo; 102 103 n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo, 104 RTE_MAX_LCORE, global_cpumaps, global_maplen); 105 if (n_vcpus < 0) { 106 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 107 "active VM '%s'\n", vm_info->name); 108 return -1; 109 } 110 update_pcpus: 111 if (n_vcpus >= RTE_MAX_LCORE) { 112 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range " 113 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1); 114 return -1; 115 } 116 if (n_vcpus != vm_info->info.nrVirtCpu) { 117 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s" 118 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu, 119 n_vcpus); 120 vm_info->info.nrVirtCpu = n_vcpus; 121 } 122 rte_spinlock_lock(&(vm_info->config_spinlock)); 123 for (i = 0; i < vm_info->info.nrVirtCpu; i++) { 124 for (j = 0; j < global_n_host_cpus; j++) { 125 if (VIR_CPU_USABLE(global_cpumaps, 126 global_maplen, i, j) <= 0) 127 continue; 128 vm_info->pcpu_map[i] = j; 129 } 130 } 131 rte_spinlock_unlock(&(vm_info->config_spinlock)); 132 return 0; 133 } 134 135 int 136 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu) 137 { 138 int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG; 139 struct virtual_machine_info *vm_info; 140 141 if (vcpu >= RTE_MAX_LCORE) { 142 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n", 143 vcpu, RTE_MAX_LCORE-1); 144 return -1; 145 } 146 147 vm_info = find_domain_by_name(vm_name); 148 if (vm_info == NULL) { 149 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 150 return -1; 151 } 152 153 if (!virDomainIsActive(vm_info->domainPtr)) { 154 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 155 " for VM '%s', VM is not active\n", 156 vcpu, vm_info->name); 157 return -1; 158 } 159 160 if (vcpu >= vm_info->info.nrVirtCpu) { 161 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of " 162 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu); 163 return -1; 164 } 165 memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen); 166 167 VIR_USE_CPU(global_cpumaps, pcpu); 168 169 if (pcpu >= global_n_host_cpus) { 170 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available " 171 "number of CPUs(%u)\n", 172 pcpu, global_n_host_cpus); 173 return -1; 174 } 175 176 if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps, 177 global_maplen, flags) < 0) { 178 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 179 " for VM '%s'\n", vcpu, 180 vm_info->name); 181 return -1; 182 } 183 rte_spinlock_lock(&(vm_info->config_spinlock)); 184 vm_info->pcpu_map[vcpu] = pcpu; 185 rte_spinlock_unlock(&(vm_info->config_spinlock)); 186 return 0; 187 } 188 189 uint16_t 190 get_pcpu(struct channel_info *chan_info, unsigned int vcpu) 191 { 192 struct virtual_machine_info *vm_info = 193 (struct virtual_machine_info *)chan_info->priv_info; 194 195 if (global_hypervisor_available && (vm_info != NULL)) { 196 uint16_t pcpu; 197 rte_spinlock_lock(&(vm_info->config_spinlock)); 198 pcpu = vm_info->pcpu_map[vcpu]; 199 rte_spinlock_unlock(&(vm_info->config_spinlock)); 200 return pcpu; 201 } else 202 return 0; 203 } 204 205 static inline int 206 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num) 207 { 208 rte_spinlock_lock(&(vm_info->config_spinlock)); 209 if (vm_info->channel_mask[channel_num] == 1) { 210 rte_spinlock_unlock(&(vm_info->config_spinlock)); 211 return 1; 212 } 213 rte_spinlock_unlock(&(vm_info->config_spinlock)); 214 return 0; 215 } 216 217 218 219 static int 220 open_non_blocking_channel(struct channel_info *info) 221 { 222 int ret, flags; 223 struct sockaddr_un sock_addr; 224 fd_set soc_fd_set; 225 struct timeval tv; 226 227 info->fd = socket(AF_UNIX, SOCK_STREAM, 0); 228 if (info->fd < 0) { 229 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n", 230 strerror(errno), 231 info->channel_path); 232 return -1; 233 } 234 sock_addr.sun_family = AF_UNIX; 235 memcpy(&sock_addr.sun_path, info->channel_path, 236 strlen(info->channel_path)+1); 237 238 /* Get current flags */ 239 flags = fcntl(info->fd, F_GETFL, 0); 240 if (flags < 0) { 241 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 242 "'%s'\n", strerror(errno), info->channel_path); 243 return 1; 244 } 245 /* Set to Non Blocking */ 246 flags |= O_NONBLOCK; 247 if (fcntl(info->fd, F_SETFL, flags) < 0) { 248 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking " 249 "socket for '%s'\n", strerror(errno), info->channel_path); 250 return -1; 251 } 252 ret = connect(info->fd, (struct sockaddr *)&sock_addr, 253 sizeof(sock_addr)); 254 if (ret < 0) { 255 /* ECONNREFUSED error is given when VM is not active */ 256 if (errno == ECONNREFUSED) { 257 RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not " 258 "activated its endpoint to channel %s\n", 259 info->channel_path); 260 return -1; 261 } 262 /* Wait for tv_sec if in progress */ 263 else if (errno == EINPROGRESS) { 264 tv.tv_sec = 2; 265 tv.tv_usec = 0; 266 FD_ZERO(&soc_fd_set); 267 FD_SET(info->fd, &soc_fd_set); 268 if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) { 269 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel " 270 "'%s'\n", info->channel_path); 271 return -1; 272 } 273 } else { 274 /* Any other error */ 275 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket" 276 " for '%s'\n", strerror(errno), info->channel_path); 277 return -1; 278 } 279 } 280 return 0; 281 } 282 283 static int 284 open_host_channel(struct channel_info *info) 285 { 286 int flags; 287 288 info->fd = open(info->channel_path, O_RDWR | O_RSYNC); 289 if (info->fd < 0) { 290 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n", 291 strerror(errno), 292 info->channel_path); 293 return -1; 294 } 295 296 /* Get current flags */ 297 flags = fcntl(info->fd, F_GETFL, 0); 298 if (flags < 0) { 299 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 300 "'%s'\n", strerror(errno), info->channel_path); 301 return 1; 302 } 303 /* Set to Non Blocking */ 304 flags |= O_NONBLOCK; 305 if (fcntl(info->fd, F_SETFL, flags) < 0) { 306 RTE_LOG(WARNING, CHANNEL_MANAGER, 307 "Error(%s) setting non-blocking " 308 "socket for '%s'\n", 309 strerror(errno), info->channel_path); 310 return -1; 311 } 312 return 0; 313 } 314 315 static int 316 setup_channel_info(struct virtual_machine_info **vm_info_dptr, 317 struct channel_info **chan_info_dptr, unsigned channel_num) 318 { 319 struct channel_info *chan_info = *chan_info_dptr; 320 struct virtual_machine_info *vm_info = *vm_info_dptr; 321 322 chan_info->channel_num = channel_num; 323 chan_info->priv_info = (void *)vm_info; 324 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 325 chan_info->type = CHANNEL_TYPE_BINARY; 326 if (open_non_blocking_channel(chan_info) < 0) { 327 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: " 328 "'%s' for VM '%s'\n", 329 chan_info->channel_path, vm_info->name); 330 return -1; 331 } 332 if (add_channel_to_monitor(&chan_info) < 0) { 333 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 334 "'%s' to epoll ctl for VM '%s'\n", 335 chan_info->channel_path, vm_info->name); 336 return -1; 337 338 } 339 rte_spinlock_lock(&(vm_info->config_spinlock)); 340 vm_info->num_channels++; 341 vm_info->channel_mask[channel_num] = 1; 342 vm_info->channels[channel_num] = chan_info; 343 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 344 rte_spinlock_unlock(&(vm_info->config_spinlock)); 345 return 0; 346 } 347 348 static int 349 fifo_path(char *dst, unsigned int len, unsigned int id) 350 { 351 int cnt; 352 353 cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH, 354 CHANNEL_MGR_FIFO_PATTERN_NAME, id); 355 356 if ((cnt < 0) || (cnt > (int)len - 1)) { 357 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper " 358 "string for fifo path\n"); 359 360 return -1; 361 } 362 363 return 0; 364 } 365 366 static int 367 setup_host_channel_info(struct channel_info **chan_info_dptr, 368 unsigned int channel_num) 369 { 370 struct channel_info *chan_info = *chan_info_dptr; 371 372 chan_info->channel_num = channel_num; 373 chan_info->priv_info = (void *)NULL; 374 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 375 chan_info->type = CHANNEL_TYPE_JSON; 376 377 if (open_host_channel(chan_info) < 0) { 378 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: " 379 "'%s'\n", 380 chan_info->channel_path); 381 return -1; 382 } 383 if (add_channel_to_monitor(&chan_info) < 0) { 384 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 385 "'%s' to epoll ctl\n", 386 chan_info->channel_path); 387 return -1; 388 389 } 390 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 391 return 0; 392 } 393 394 int 395 add_all_channels(const char *vm_name) 396 { 397 DIR *d; 398 struct dirent *dir; 399 struct virtual_machine_info *vm_info; 400 struct channel_info *chan_info; 401 char *token, *remaining, *tail_ptr; 402 char socket_name[PATH_MAX]; 403 unsigned channel_num; 404 int num_channels_enabled = 0; 405 406 /* verify VM exists */ 407 vm_info = find_domain_by_name(vm_name); 408 if (vm_info == NULL) { 409 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found" 410 " during channel discovery\n", vm_name); 411 return 0; 412 } 413 if (!virDomainIsActive(vm_info->domainPtr)) { 414 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 415 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 416 return 0; 417 } 418 d = opendir(CHANNEL_MGR_SOCKET_PATH); 419 if (d == NULL) { 420 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n", 421 CHANNEL_MGR_SOCKET_PATH, strerror(errno)); 422 return -1; 423 } 424 while ((dir = readdir(d)) != NULL) { 425 if (!strncmp(dir->d_name, ".", 1) || 426 !strncmp(dir->d_name, "..", 2)) 427 continue; 428 429 strlcpy(socket_name, dir->d_name, sizeof(socket_name)); 430 remaining = socket_name; 431 /* Extract vm_name from "<vm_name>.<channel_num>" */ 432 token = strsep(&remaining, "."); 433 if (remaining == NULL) 434 continue; 435 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN)) 436 continue; 437 438 /* remaining should contain only <channel_num> */ 439 errno = 0; 440 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0); 441 if ((errno != 0) || (remaining[0] == '\0') || 442 tail_ptr == NULL || (*tail_ptr != '\0')) { 443 RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name" 444 "'%s' found it should be in the form of " 445 "'<guest_name>.<channel_num>(decimal)'\n", 446 dir->d_name); 447 continue; 448 } 449 if (channel_num >= RTE_MAX_LCORE) { 450 RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is " 451 "greater than max allowable: %d, skipping '%s%s'\n", 452 channel_num, RTE_MAX_LCORE-1, 453 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 454 continue; 455 } 456 /* if channel has not been added previously */ 457 if (channel_exists(vm_info, channel_num)) 458 continue; 459 460 chan_info = rte_malloc(NULL, sizeof(*chan_info), 461 RTE_CACHE_LINE_SIZE); 462 if (chan_info == NULL) { 463 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 464 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name); 465 continue; 466 } 467 468 snprintf(chan_info->channel_path, 469 sizeof(chan_info->channel_path), "%s%s", 470 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 471 472 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) { 473 rte_free(chan_info); 474 continue; 475 } 476 477 num_channels_enabled++; 478 } 479 closedir(d); 480 return num_channels_enabled; 481 } 482 483 int 484 add_channels(const char *vm_name, unsigned *channel_list, 485 unsigned len_channel_list) 486 { 487 struct virtual_machine_info *vm_info; 488 struct channel_info *chan_info; 489 char socket_path[PATH_MAX]; 490 unsigned i; 491 int num_channels_enabled = 0; 492 493 vm_info = find_domain_by_name(vm_name); 494 if (vm_info == NULL) { 495 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 496 "not found\n", vm_name); 497 return 0; 498 } 499 500 if (!virDomainIsActive(vm_info->domainPtr)) { 501 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 502 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 503 return 0; 504 } 505 506 for (i = 0; i < len_channel_list; i++) { 507 508 if (channel_list[i] >= RTE_MAX_LCORE) { 509 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range " 510 "0...%d\n", channel_list[i], 511 RTE_MAX_LCORE-1); 512 continue; 513 } 514 if (channel_exists(vm_info, channel_list[i])) { 515 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping " 516 "'%s.%u'\n", vm_name, i); 517 continue; 518 } 519 520 snprintf(socket_path, sizeof(socket_path), "%s%s.%u", 521 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 522 errno = 0; 523 if (access(socket_path, F_OK) < 0) { 524 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: " 525 "%s\n", socket_path, strerror(errno)); 526 continue; 527 } 528 chan_info = rte_malloc(NULL, sizeof(*chan_info), 529 RTE_CACHE_LINE_SIZE); 530 if (chan_info == NULL) { 531 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 532 "channel '%s'\n", socket_path); 533 continue; 534 } 535 snprintf(chan_info->channel_path, 536 sizeof(chan_info->channel_path), "%s%s.%u", 537 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 538 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) { 539 rte_free(chan_info); 540 continue; 541 } 542 num_channels_enabled++; 543 544 } 545 return num_channels_enabled; 546 } 547 548 int 549 add_host_channels(void) 550 { 551 struct channel_info *chan_info; 552 char socket_path[PATH_MAX]; 553 int num_channels_enabled = 0; 554 int ret; 555 struct core_info *ci; 556 struct channel_info *chan_infos[RTE_MAX_LCORE]; 557 int i; 558 559 for (i = 0; i < RTE_MAX_LCORE; i++) 560 chan_infos[i] = NULL; 561 562 ci = get_core_info(); 563 if (ci == NULL) { 564 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n"); 565 return 0; 566 } 567 568 for (i = 0; i < ci->core_count; i++) { 569 if (ci->cd[i].global_enabled_cpus == 0) 570 continue; 571 572 ret = fifo_path(socket_path, sizeof(socket_path), i); 573 if (ret < 0) 574 goto error; 575 576 ret = mkfifo(socket_path, 0660); 577 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n", 578 socket_path); 579 if ((errno != EEXIST) && (ret < 0)) { 580 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: " 581 "%s\n", socket_path, strerror(errno)); 582 goto error; 583 } 584 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0); 585 if (chan_info == NULL) { 586 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 587 "channel '%s'\n", socket_path); 588 goto error; 589 } 590 chan_infos[i] = chan_info; 591 strlcpy(chan_info->channel_path, socket_path, 592 sizeof(chan_info->channel_path)); 593 594 if (setup_host_channel_info(&chan_info, i) < 0) { 595 rte_free(chan_info); 596 chan_infos[i] = NULL; 597 goto error; 598 } 599 num_channels_enabled++; 600 } 601 602 return num_channels_enabled; 603 error: 604 /* Clean up the channels opened before we hit an error. */ 605 for (i = 0; i < ci->core_count; i++) { 606 if (chan_infos[i] != NULL) { 607 remove_channel_from_monitor(chan_infos[i]); 608 close(chan_infos[i]->fd); 609 rte_free(chan_infos[i]); 610 } 611 } 612 return 0; 613 } 614 615 int 616 remove_channel(struct channel_info **chan_info_dptr) 617 { 618 struct virtual_machine_info *vm_info; 619 struct channel_info *chan_info = *chan_info_dptr; 620 621 close(chan_info->fd); 622 623 vm_info = (struct virtual_machine_info *)chan_info->priv_info; 624 625 rte_spinlock_lock(&(vm_info->config_spinlock)); 626 vm_info->channel_mask[chan_info->channel_num] = 0; 627 vm_info->num_channels--; 628 rte_spinlock_unlock(&(vm_info->config_spinlock)); 629 630 rte_free(chan_info); 631 return 0; 632 } 633 634 int 635 set_channel_status_all(const char *vm_name, enum channel_status status) 636 { 637 struct virtual_machine_info *vm_info; 638 unsigned i; 639 char mask[RTE_MAX_LCORE]; 640 int num_channels_changed = 0; 641 642 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 643 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 644 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 645 "disabled: Unable to change status for VM '%s'\n", vm_name); 646 } 647 vm_info = find_domain_by_name(vm_name); 648 if (vm_info == NULL) { 649 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' " 650 "not found\n", vm_name); 651 return 0; 652 } 653 654 rte_spinlock_lock(&(vm_info->config_spinlock)); 655 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 656 for (i = 0; i < RTE_MAX_LCORE; i++) { 657 if (mask[i] != 1) 658 continue; 659 vm_info->channels[i]->status = status; 660 num_channels_changed++; 661 } 662 rte_spinlock_unlock(&(vm_info->config_spinlock)); 663 return num_channels_changed; 664 665 } 666 667 int 668 set_channel_status(const char *vm_name, unsigned *channel_list, 669 unsigned len_channel_list, enum channel_status status) 670 { 671 struct virtual_machine_info *vm_info; 672 unsigned i; 673 int num_channels_changed = 0; 674 675 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 676 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 677 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 678 "disabled: Unable to change status for VM '%s'\n", vm_name); 679 } 680 vm_info = find_domain_by_name(vm_name); 681 if (vm_info == NULL) { 682 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 683 "not found\n", vm_name); 684 return 0; 685 } 686 for (i = 0; i < len_channel_list; i++) { 687 if (channel_exists(vm_info, channel_list[i])) { 688 rte_spinlock_lock(&(vm_info->config_spinlock)); 689 vm_info->channels[channel_list[i]]->status = status; 690 rte_spinlock_unlock(&(vm_info->config_spinlock)); 691 num_channels_changed++; 692 } 693 } 694 return num_channels_changed; 695 } 696 697 void 698 get_all_vm(int *num_vm, int *num_vcpu) 699 { 700 701 virNodeInfo node_info; 702 virDomainPtr *domptr; 703 int i, ii, numVcpus[MAX_VCPUS], n_vcpus; 704 unsigned int jj; 705 const char *vm_name; 706 unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | 707 VIR_CONNECT_LIST_DOMAINS_PERSISTENT; 708 unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; 709 710 if (!global_hypervisor_available) 711 return; 712 713 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 714 if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { 715 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 716 return; 717 } 718 719 /* Returns number of pcpus */ 720 global_n_host_cpus = (unsigned int)node_info.cpus; 721 722 /* Returns number of active domains */ 723 *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, 724 domain_flags); 725 if (*num_vm <= 0) { 726 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); 727 return; 728 } 729 730 for (i = 0; i < *num_vm; i++) { 731 732 /* Get Domain Names */ 733 vm_name = virDomainGetName(domptr[i]); 734 lvm_info[i].vm_name = vm_name; 735 736 /* Get Number of Vcpus */ 737 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); 738 739 /* Get Number of VCpus & VcpuPinInfo */ 740 n_vcpus = virDomainGetVcpuPinInfo(domptr[i], 741 numVcpus[i], global_cpumaps, 742 global_maplen, domain_flag); 743 744 if ((int)n_vcpus > 0) { 745 *num_vcpu = n_vcpus; 746 lvm_info[i].num_cpus = n_vcpus; 747 } 748 749 /* Save pcpu in use by libvirt VMs */ 750 for (ii = 0; ii < n_vcpus; ii++) { 751 for (jj = 0; jj < global_n_host_cpus; jj++) { 752 if (VIR_CPU_USABLE(global_cpumaps, 753 global_maplen, ii, jj) > 0) { 754 lvm_info[i].pcpus[ii] = jj; 755 } 756 } 757 } 758 } 759 } 760 761 int 762 get_info_vm(const char *vm_name, struct vm_info *info) 763 { 764 struct virtual_machine_info *vm_info; 765 unsigned i, channel_num = 0; 766 char mask[RTE_MAX_LCORE]; 767 768 vm_info = find_domain_by_name(vm_name); 769 if (vm_info == NULL) { 770 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 771 return -1; 772 } 773 info->status = CHANNEL_MGR_VM_ACTIVE; 774 if (!virDomainIsActive(vm_info->domainPtr)) 775 info->status = CHANNEL_MGR_VM_INACTIVE; 776 777 rte_spinlock_lock(&(vm_info->config_spinlock)); 778 779 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 780 for (i = 0; i < RTE_MAX_LCORE; i++) { 781 if (mask[i] != 1) 782 continue; 783 info->channels[channel_num].channel_num = i; 784 memcpy(info->channels[channel_num].channel_path, 785 vm_info->channels[i]->channel_path, 786 UNIX_PATH_MAX); 787 info->channels[channel_num].status = 788 vm_info->channels[i]->status; 789 info->channels[channel_num].fd = 790 vm_info->channels[i]->fd; 791 channel_num++; 792 } 793 794 info->num_channels = channel_num; 795 info->num_vcpus = vm_info->info.nrVirtCpu; 796 rte_spinlock_unlock(&(vm_info->config_spinlock)); 797 798 memcpy(info->name, vm_info->name, sizeof(vm_info->name)); 799 rte_spinlock_lock(&(vm_info->config_spinlock)); 800 for (i = 0; i < info->num_vcpus; i++) { 801 info->pcpu_map[i] = vm_info->pcpu_map[i]; 802 } 803 rte_spinlock_unlock(&(vm_info->config_spinlock)); 804 return 0; 805 } 806 807 int 808 add_vm(const char *vm_name) 809 { 810 struct virtual_machine_info *new_domain; 811 virDomainPtr dom_ptr; 812 int i; 813 814 if (find_domain_by_name(vm_name) != NULL) { 815 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' " 816 "already exists\n", vm_name); 817 return -1; 818 } 819 820 if (global_vir_conn_ptr == NULL) { 821 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n"); 822 return -1; 823 } 824 dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name); 825 if (dom_ptr == NULL) { 826 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: " 827 "VM '%s' not found\n", vm_name); 828 return -1; 829 } 830 831 new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain), 832 RTE_CACHE_LINE_SIZE); 833 if (new_domain == NULL) { 834 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM " 835 "info\n"); 836 return -1; 837 } 838 new_domain->domainPtr = dom_ptr; 839 if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) { 840 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n"); 841 rte_free(new_domain); 842 return -1; 843 } 844 if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) { 845 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is " 846 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu, 847 RTE_MAX_LCORE); 848 rte_free(new_domain); 849 return -1; 850 } 851 852 for (i = 0; i < RTE_MAX_LCORE; i++) 853 new_domain->pcpu_map[i] = 0; 854 855 if (update_pcpus_mask(new_domain) < 0) { 856 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n"); 857 rte_free(new_domain); 858 return -1; 859 } 860 strncpy(new_domain->name, vm_name, sizeof(new_domain->name)); 861 new_domain->name[sizeof(new_domain->name) - 1] = '\0'; 862 memset(new_domain->channel_mask, 0, RTE_MAX_LCORE); 863 new_domain->num_channels = 0; 864 865 if (!virDomainIsActive(dom_ptr)) 866 new_domain->status = CHANNEL_MGR_VM_INACTIVE; 867 else 868 new_domain->status = CHANNEL_MGR_VM_ACTIVE; 869 870 rte_spinlock_init(&(new_domain->config_spinlock)); 871 LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info); 872 return 0; 873 } 874 875 int 876 remove_vm(const char *vm_name) 877 { 878 struct virtual_machine_info *vm_info = find_domain_by_name(vm_name); 879 880 if (vm_info == NULL) { 881 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' " 882 "not found\n", vm_name); 883 return -1; 884 } 885 rte_spinlock_lock(&vm_info->config_spinlock); 886 if (vm_info->num_channels != 0) { 887 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are " 888 "%"PRId8" channels still active\n", 889 vm_name, vm_info->num_channels); 890 rte_spinlock_unlock(&vm_info->config_spinlock); 891 return -1; 892 } 893 LIST_REMOVE(vm_info, vms_info); 894 rte_spinlock_unlock(&vm_info->config_spinlock); 895 rte_free(vm_info); 896 return 0; 897 } 898 899 static void 900 disconnect_hypervisor(void) 901 { 902 if (global_vir_conn_ptr != NULL) { 903 virConnectClose(global_vir_conn_ptr); 904 global_vir_conn_ptr = NULL; 905 } 906 } 907 908 static int 909 connect_hypervisor(const char *path) 910 { 911 if (global_vir_conn_ptr != NULL) { 912 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection " 913 "already established\n", path); 914 return -1; 915 } 916 global_vir_conn_ptr = virConnectOpen(path); 917 if (global_vir_conn_ptr == NULL) { 918 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to " 919 "Hypervisor '%s'\n", path); 920 return -1; 921 } 922 return 0; 923 } 924 int 925 channel_manager_init(const char *path __rte_unused) 926 { 927 virNodeInfo info; 928 929 LIST_INIT(&vm_list_head); 930 if (connect_hypervisor(path) < 0) { 931 global_n_host_cpus = 64; 932 global_hypervisor_available = 0; 933 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n"); 934 } else { 935 global_hypervisor_available = 1; 936 937 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE); 938 939 global_vircpuinfo = rte_zmalloc(NULL, 940 sizeof(*global_vircpuinfo) * 941 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE); 942 if (global_vircpuinfo == NULL) { 943 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n"); 944 goto error; 945 } 946 global_cpumaps = rte_zmalloc(NULL, 947 RTE_MAX_LCORE * global_maplen, 948 RTE_CACHE_LINE_SIZE); 949 if (global_cpumaps == NULL) 950 goto error; 951 952 if (virNodeGetInfo(global_vir_conn_ptr, &info)) { 953 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 954 goto error; 955 } 956 global_n_host_cpus = (unsigned int)info.cpus; 957 } 958 959 960 961 if (global_n_host_cpus > RTE_MAX_LCORE) { 962 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the " 963 "maximum of %u. No cores over %u should be used.\n", 964 global_n_host_cpus, RTE_MAX_LCORE, 965 RTE_MAX_LCORE - 1); 966 global_n_host_cpus = RTE_MAX_LCORE; 967 } 968 969 return 0; 970 error: 971 if (global_hypervisor_available) 972 disconnect_hypervisor(); 973 return -1; 974 } 975 976 void 977 channel_manager_exit(void) 978 { 979 unsigned i; 980 char mask[RTE_MAX_LCORE]; 981 struct virtual_machine_info *vm_info; 982 983 LIST_FOREACH(vm_info, &vm_list_head, vms_info) { 984 985 rte_spinlock_lock(&(vm_info->config_spinlock)); 986 987 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 988 for (i = 0; i < RTE_MAX_LCORE; i++) { 989 if (mask[i] != 1) 990 continue; 991 remove_channel_from_monitor( 992 vm_info->channels[i]); 993 close(vm_info->channels[i]->fd); 994 rte_free(vm_info->channels[i]); 995 } 996 rte_spinlock_unlock(&(vm_info->config_spinlock)); 997 998 LIST_REMOVE(vm_info, vms_info); 999 rte_free(vm_info); 1000 } 1001 1002 if (global_hypervisor_available) { 1003 /* Only needed if hypervisor available */ 1004 rte_free(global_cpumaps); 1005 rte_free(global_vircpuinfo); 1006 disconnect_hypervisor(); 1007 } 1008 } 1009