1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <fcntl.h> 8 #include <unistd.h> 9 #include <inttypes.h> 10 #include <dirent.h> 11 #include <errno.h> 12 13 #include <sys/queue.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 #include <sys/socket.h> 17 #include <sys/select.h> 18 19 #include <rte_string_fns.h> 20 #include <rte_malloc.h> 21 #include <rte_memory.h> 22 #include <rte_mempool.h> 23 #include <rte_log.h> 24 #include <rte_atomic.h> 25 #include <rte_spinlock.h> 26 27 #include <libvirt/libvirt.h> 28 29 #include "channel_manager.h" 30 #include "channel_commands.h" 31 #include "channel_monitor.h" 32 #include "power_manager.h" 33 34 35 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1 36 37 struct libvirt_vm_info lvm_info[MAX_CLIENTS]; 38 39 /* Global pointer to libvirt connection */ 40 static virConnectPtr global_vir_conn_ptr; 41 42 static unsigned char *global_cpumaps; 43 static virVcpuInfo *global_vircpuinfo; 44 static size_t global_maplen; 45 46 static unsigned int global_n_host_cpus; 47 static bool global_hypervisor_available; 48 49 /* 50 * Represents a single Virtual Machine 51 */ 52 struct virtual_machine_info { 53 char name[CHANNEL_MGR_MAX_NAME_LEN]; 54 uint16_t pcpu_map[RTE_MAX_LCORE]; 55 struct channel_info *channels[RTE_MAX_LCORE]; 56 char channel_mask[RTE_MAX_LCORE]; 57 uint8_t num_channels; 58 enum vm_status status; 59 virDomainPtr domainPtr; 60 virDomainInfo info; 61 rte_spinlock_t config_spinlock; 62 int allow_query; 63 LIST_ENTRY(virtual_machine_info) vms_info; 64 }; 65 66 LIST_HEAD(, virtual_machine_info) vm_list_head; 67 68 static struct virtual_machine_info * 69 find_domain_by_name(const char *name) 70 { 71 struct virtual_machine_info *info; 72 LIST_FOREACH(info, &vm_list_head, vms_info) { 73 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1)) 74 return info; 75 } 76 return NULL; 77 } 78 79 static int 80 update_pcpus_mask(struct virtual_machine_info *vm_info) 81 { 82 virVcpuInfoPtr cpuinfo; 83 unsigned i, j; 84 int n_vcpus; 85 86 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 87 88 if (!virDomainIsActive(vm_info->domainPtr)) { 89 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr, 90 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen, 91 VIR_DOMAIN_AFFECT_CONFIG); 92 if (n_vcpus < 0) { 93 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 94 "in-active VM '%s'\n", vm_info->name); 95 return -1; 96 } 97 goto update_pcpus; 98 } 99 100 memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)* 101 RTE_MAX_LCORE); 102 103 cpuinfo = global_vircpuinfo; 104 105 n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo, 106 RTE_MAX_LCORE, global_cpumaps, global_maplen); 107 if (n_vcpus < 0) { 108 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 109 "active VM '%s'\n", vm_info->name); 110 return -1; 111 } 112 update_pcpus: 113 if (n_vcpus >= RTE_MAX_LCORE) { 114 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range " 115 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1); 116 return -1; 117 } 118 if (n_vcpus != vm_info->info.nrVirtCpu) { 119 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s" 120 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu, 121 n_vcpus); 122 vm_info->info.nrVirtCpu = n_vcpus; 123 } 124 rte_spinlock_lock(&(vm_info->config_spinlock)); 125 for (i = 0; i < vm_info->info.nrVirtCpu; i++) { 126 for (j = 0; j < global_n_host_cpus; j++) { 127 if (VIR_CPU_USABLE(global_cpumaps, 128 global_maplen, i, j) <= 0) 129 continue; 130 vm_info->pcpu_map[i] = j; 131 } 132 } 133 rte_spinlock_unlock(&(vm_info->config_spinlock)); 134 return 0; 135 } 136 137 int 138 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu) 139 { 140 int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG; 141 struct virtual_machine_info *vm_info; 142 143 if (vcpu >= RTE_MAX_LCORE) { 144 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n", 145 vcpu, RTE_MAX_LCORE-1); 146 return -1; 147 } 148 149 vm_info = find_domain_by_name(vm_name); 150 if (vm_info == NULL) { 151 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 152 return -1; 153 } 154 155 if (!virDomainIsActive(vm_info->domainPtr)) { 156 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 157 " for VM '%s', VM is not active\n", 158 vcpu, vm_info->name); 159 return -1; 160 } 161 162 if (vcpu >= vm_info->info.nrVirtCpu) { 163 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of " 164 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu); 165 return -1; 166 } 167 memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen); 168 169 VIR_USE_CPU(global_cpumaps, pcpu); 170 171 if (pcpu >= global_n_host_cpus) { 172 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available " 173 "number of CPUs(%u)\n", 174 pcpu, global_n_host_cpus); 175 return -1; 176 } 177 178 if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps, 179 global_maplen, flags) < 0) { 180 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 181 " for VM '%s'\n", vcpu, 182 vm_info->name); 183 return -1; 184 } 185 rte_spinlock_lock(&(vm_info->config_spinlock)); 186 vm_info->pcpu_map[vcpu] = pcpu; 187 rte_spinlock_unlock(&(vm_info->config_spinlock)); 188 return 0; 189 } 190 191 uint16_t 192 get_pcpu(struct channel_info *chan_info, unsigned int vcpu) 193 { 194 struct virtual_machine_info *vm_info = 195 (struct virtual_machine_info *)chan_info->priv_info; 196 197 if (global_hypervisor_available && (vm_info != NULL)) { 198 uint16_t pcpu; 199 rte_spinlock_lock(&(vm_info->config_spinlock)); 200 pcpu = vm_info->pcpu_map[vcpu]; 201 rte_spinlock_unlock(&(vm_info->config_spinlock)); 202 return pcpu; 203 } else 204 return 0; 205 } 206 207 static inline int 208 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num) 209 { 210 rte_spinlock_lock(&(vm_info->config_spinlock)); 211 if (vm_info->channel_mask[channel_num] == 1) { 212 rte_spinlock_unlock(&(vm_info->config_spinlock)); 213 return 1; 214 } 215 rte_spinlock_unlock(&(vm_info->config_spinlock)); 216 return 0; 217 } 218 219 220 221 static int 222 open_non_blocking_channel(struct channel_info *info) 223 { 224 int ret, flags; 225 struct sockaddr_un sock_addr; 226 fd_set soc_fd_set; 227 struct timeval tv; 228 229 info->fd = socket(AF_UNIX, SOCK_STREAM, 0); 230 if (info->fd < 0) { 231 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n", 232 strerror(errno), 233 info->channel_path); 234 return -1; 235 } 236 sock_addr.sun_family = AF_UNIX; 237 memcpy(&sock_addr.sun_path, info->channel_path, 238 strlen(info->channel_path)+1); 239 240 /* Get current flags */ 241 flags = fcntl(info->fd, F_GETFL, 0); 242 if (flags < 0) { 243 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 244 "'%s'\n", strerror(errno), info->channel_path); 245 return 1; 246 } 247 /* Set to Non Blocking */ 248 flags |= O_NONBLOCK; 249 if (fcntl(info->fd, F_SETFL, flags) < 0) { 250 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking " 251 "socket for '%s'\n", strerror(errno), info->channel_path); 252 return -1; 253 } 254 ret = connect(info->fd, (struct sockaddr *)&sock_addr, 255 sizeof(sock_addr)); 256 if (ret < 0) { 257 /* ECONNREFUSED error is given when VM is not active */ 258 if (errno == ECONNREFUSED) { 259 RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not " 260 "activated its endpoint to channel %s\n", 261 info->channel_path); 262 return -1; 263 } 264 /* Wait for tv_sec if in progress */ 265 else if (errno == EINPROGRESS) { 266 tv.tv_sec = 2; 267 tv.tv_usec = 0; 268 FD_ZERO(&soc_fd_set); 269 FD_SET(info->fd, &soc_fd_set); 270 if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) { 271 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel " 272 "'%s'\n", info->channel_path); 273 return -1; 274 } 275 } else { 276 /* Any other error */ 277 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket" 278 " for '%s'\n", strerror(errno), info->channel_path); 279 return -1; 280 } 281 } 282 return 0; 283 } 284 285 static int 286 open_host_channel(struct channel_info *info) 287 { 288 int flags; 289 290 info->fd = open(info->channel_path, O_RDWR | O_RSYNC); 291 if (info->fd < 0) { 292 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n", 293 strerror(errno), 294 info->channel_path); 295 return -1; 296 } 297 298 /* Get current flags */ 299 flags = fcntl(info->fd, F_GETFL, 0); 300 if (flags < 0) { 301 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 302 "'%s'\n", strerror(errno), info->channel_path); 303 return 1; 304 } 305 /* Set to Non Blocking */ 306 flags |= O_NONBLOCK; 307 if (fcntl(info->fd, F_SETFL, flags) < 0) { 308 RTE_LOG(WARNING, CHANNEL_MANAGER, 309 "Error(%s) setting non-blocking " 310 "socket for '%s'\n", 311 strerror(errno), info->channel_path); 312 return -1; 313 } 314 return 0; 315 } 316 317 static int 318 setup_channel_info(struct virtual_machine_info **vm_info_dptr, 319 struct channel_info **chan_info_dptr, unsigned channel_num) 320 { 321 struct channel_info *chan_info = *chan_info_dptr; 322 struct virtual_machine_info *vm_info = *vm_info_dptr; 323 324 chan_info->channel_num = channel_num; 325 chan_info->priv_info = (void *)vm_info; 326 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 327 chan_info->type = CHANNEL_TYPE_BINARY; 328 if (open_non_blocking_channel(chan_info) < 0) { 329 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: " 330 "'%s' for VM '%s'\n", 331 chan_info->channel_path, vm_info->name); 332 return -1; 333 } 334 if (add_channel_to_monitor(&chan_info) < 0) { 335 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 336 "'%s' to epoll ctl for VM '%s'\n", 337 chan_info->channel_path, vm_info->name); 338 return -1; 339 340 } 341 rte_spinlock_lock(&(vm_info->config_spinlock)); 342 vm_info->num_channels++; 343 vm_info->channel_mask[channel_num] = 1; 344 vm_info->channels[channel_num] = chan_info; 345 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 346 rte_spinlock_unlock(&(vm_info->config_spinlock)); 347 return 0; 348 } 349 350 static int 351 fifo_path(char *dst, unsigned int len, unsigned int id) 352 { 353 int cnt; 354 355 cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH, 356 CHANNEL_MGR_FIFO_PATTERN_NAME, id); 357 358 if ((cnt < 0) || (cnt > (int)len - 1)) { 359 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper " 360 "string for fifo path\n"); 361 362 return -1; 363 } 364 365 return 0; 366 } 367 368 static int 369 setup_host_channel_info(struct channel_info **chan_info_dptr, 370 unsigned int channel_num) 371 { 372 struct channel_info *chan_info = *chan_info_dptr; 373 374 chan_info->channel_num = channel_num; 375 chan_info->priv_info = (void *)NULL; 376 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 377 chan_info->type = CHANNEL_TYPE_JSON; 378 379 if (open_host_channel(chan_info) < 0) { 380 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: " 381 "'%s'\n", 382 chan_info->channel_path); 383 return -1; 384 } 385 if (add_channel_to_monitor(&chan_info) < 0) { 386 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 387 "'%s' to epoll ctl\n", 388 chan_info->channel_path); 389 return -1; 390 391 } 392 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 393 return 0; 394 } 395 396 int 397 add_all_channels(const char *vm_name) 398 { 399 DIR *d; 400 struct dirent *dir; 401 struct virtual_machine_info *vm_info; 402 struct channel_info *chan_info; 403 char *token, *remaining, *tail_ptr; 404 char socket_name[PATH_MAX]; 405 unsigned channel_num; 406 int num_channels_enabled = 0; 407 408 /* verify VM exists */ 409 vm_info = find_domain_by_name(vm_name); 410 if (vm_info == NULL) { 411 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found" 412 " during channel discovery\n", vm_name); 413 return 0; 414 } 415 if (!virDomainIsActive(vm_info->domainPtr)) { 416 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 417 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 418 return 0; 419 } 420 d = opendir(CHANNEL_MGR_SOCKET_PATH); 421 if (d == NULL) { 422 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n", 423 CHANNEL_MGR_SOCKET_PATH, strerror(errno)); 424 return -1; 425 } 426 while ((dir = readdir(d)) != NULL) { 427 if (!strncmp(dir->d_name, ".", 1) || 428 !strncmp(dir->d_name, "..", 2)) 429 continue; 430 431 strlcpy(socket_name, dir->d_name, sizeof(socket_name)); 432 remaining = socket_name; 433 /* Extract vm_name from "<vm_name>.<channel_num>" */ 434 token = strsep(&remaining, "."); 435 if (remaining == NULL) 436 continue; 437 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN)) 438 continue; 439 440 /* remaining should contain only <channel_num> */ 441 errno = 0; 442 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0); 443 if ((errno != 0) || (remaining[0] == '\0') || 444 tail_ptr == NULL || (*tail_ptr != '\0')) { 445 RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name" 446 "'%s' found it should be in the form of " 447 "'<guest_name>.<channel_num>(decimal)'\n", 448 dir->d_name); 449 continue; 450 } 451 if (channel_num >= RTE_MAX_LCORE) { 452 RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is " 453 "greater than max allowable: %d, skipping '%s%s'\n", 454 channel_num, RTE_MAX_LCORE-1, 455 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 456 continue; 457 } 458 /* if channel has not been added previously */ 459 if (channel_exists(vm_info, channel_num)) 460 continue; 461 462 chan_info = rte_malloc(NULL, sizeof(*chan_info), 463 RTE_CACHE_LINE_SIZE); 464 if (chan_info == NULL) { 465 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 466 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name); 467 continue; 468 } 469 470 snprintf(chan_info->channel_path, 471 sizeof(chan_info->channel_path), "%s%s", 472 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 473 474 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) { 475 rte_free(chan_info); 476 continue; 477 } 478 479 num_channels_enabled++; 480 } 481 closedir(d); 482 return num_channels_enabled; 483 } 484 485 int 486 add_channels(const char *vm_name, unsigned *channel_list, 487 unsigned len_channel_list) 488 { 489 struct virtual_machine_info *vm_info; 490 struct channel_info *chan_info; 491 char socket_path[PATH_MAX]; 492 unsigned i; 493 int num_channels_enabled = 0; 494 495 vm_info = find_domain_by_name(vm_name); 496 if (vm_info == NULL) { 497 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 498 "not found\n", vm_name); 499 return 0; 500 } 501 502 if (!virDomainIsActive(vm_info->domainPtr)) { 503 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 504 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 505 return 0; 506 } 507 508 for (i = 0; i < len_channel_list; i++) { 509 510 if (channel_list[i] >= RTE_MAX_LCORE) { 511 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range " 512 "0...%d\n", channel_list[i], 513 RTE_MAX_LCORE-1); 514 continue; 515 } 516 if (channel_exists(vm_info, channel_list[i])) { 517 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping " 518 "'%s.%u'\n", vm_name, i); 519 continue; 520 } 521 522 snprintf(socket_path, sizeof(socket_path), "%s%s.%u", 523 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 524 errno = 0; 525 if (access(socket_path, F_OK) < 0) { 526 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: " 527 "%s\n", socket_path, strerror(errno)); 528 continue; 529 } 530 chan_info = rte_malloc(NULL, sizeof(*chan_info), 531 RTE_CACHE_LINE_SIZE); 532 if (chan_info == NULL) { 533 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 534 "channel '%s'\n", socket_path); 535 continue; 536 } 537 snprintf(chan_info->channel_path, 538 sizeof(chan_info->channel_path), "%s%s.%u", 539 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 540 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) { 541 rte_free(chan_info); 542 continue; 543 } 544 num_channels_enabled++; 545 546 } 547 return num_channels_enabled; 548 } 549 550 int 551 add_host_channels(void) 552 { 553 struct channel_info *chan_info; 554 char socket_path[PATH_MAX]; 555 int num_channels_enabled = 0; 556 int ret; 557 struct core_info *ci; 558 struct channel_info *chan_infos[RTE_MAX_LCORE]; 559 int i; 560 561 for (i = 0; i < RTE_MAX_LCORE; i++) 562 chan_infos[i] = NULL; 563 564 ci = get_core_info(); 565 if (ci == NULL) { 566 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n"); 567 return 0; 568 } 569 570 for (i = 0; i < ci->core_count; i++) { 571 if (ci->cd[i].global_enabled_cpus == 0) 572 continue; 573 574 ret = fifo_path(socket_path, sizeof(socket_path), i); 575 if (ret < 0) 576 goto error; 577 578 ret = mkfifo(socket_path, 0660); 579 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n", 580 socket_path); 581 if ((errno != EEXIST) && (ret < 0)) { 582 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: " 583 "%s\n", socket_path, strerror(errno)); 584 goto error; 585 } 586 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0); 587 if (chan_info == NULL) { 588 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 589 "channel '%s'\n", socket_path); 590 goto error; 591 } 592 chan_infos[i] = chan_info; 593 strlcpy(chan_info->channel_path, socket_path, 594 sizeof(chan_info->channel_path)); 595 596 if (setup_host_channel_info(&chan_info, i) < 0) { 597 rte_free(chan_info); 598 chan_infos[i] = NULL; 599 goto error; 600 } 601 num_channels_enabled++; 602 } 603 604 return num_channels_enabled; 605 error: 606 /* Clean up the channels opened before we hit an error. */ 607 for (i = 0; i < ci->core_count; i++) { 608 if (chan_infos[i] != NULL) { 609 remove_channel_from_monitor(chan_infos[i]); 610 close(chan_infos[i]->fd); 611 rte_free(chan_infos[i]); 612 } 613 } 614 return 0; 615 } 616 617 int 618 remove_channel(struct channel_info **chan_info_dptr) 619 { 620 struct virtual_machine_info *vm_info; 621 struct channel_info *chan_info = *chan_info_dptr; 622 623 close(chan_info->fd); 624 625 vm_info = (struct virtual_machine_info *)chan_info->priv_info; 626 627 rte_spinlock_lock(&(vm_info->config_spinlock)); 628 vm_info->channel_mask[chan_info->channel_num] = 0; 629 vm_info->num_channels--; 630 rte_spinlock_unlock(&(vm_info->config_spinlock)); 631 632 rte_free(chan_info); 633 return 0; 634 } 635 636 int 637 set_channel_status_all(const char *vm_name, enum channel_status status) 638 { 639 struct virtual_machine_info *vm_info; 640 unsigned i; 641 char mask[RTE_MAX_LCORE]; 642 int num_channels_changed = 0; 643 644 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 645 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 646 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 647 "disabled: Unable to change status for VM '%s'\n", vm_name); 648 } 649 vm_info = find_domain_by_name(vm_name); 650 if (vm_info == NULL) { 651 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' " 652 "not found\n", vm_name); 653 return 0; 654 } 655 656 rte_spinlock_lock(&(vm_info->config_spinlock)); 657 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 658 for (i = 0; i < RTE_MAX_LCORE; i++) { 659 if (mask[i] != 1) 660 continue; 661 vm_info->channels[i]->status = status; 662 num_channels_changed++; 663 } 664 rte_spinlock_unlock(&(vm_info->config_spinlock)); 665 return num_channels_changed; 666 667 } 668 669 int 670 set_channel_status(const char *vm_name, unsigned *channel_list, 671 unsigned len_channel_list, enum channel_status status) 672 { 673 struct virtual_machine_info *vm_info; 674 unsigned i; 675 int num_channels_changed = 0; 676 677 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 678 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 679 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 680 "disabled: Unable to change status for VM '%s'\n", vm_name); 681 } 682 vm_info = find_domain_by_name(vm_name); 683 if (vm_info == NULL) { 684 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 685 "not found\n", vm_name); 686 return 0; 687 } 688 for (i = 0; i < len_channel_list; i++) { 689 if (channel_exists(vm_info, channel_list[i])) { 690 rte_spinlock_lock(&(vm_info->config_spinlock)); 691 vm_info->channels[channel_list[i]]->status = status; 692 rte_spinlock_unlock(&(vm_info->config_spinlock)); 693 num_channels_changed++; 694 } 695 } 696 return num_channels_changed; 697 } 698 699 void 700 get_all_vm(int *num_vm, int *num_vcpu) 701 { 702 703 virNodeInfo node_info; 704 virDomainPtr *domptr; 705 int i, ii, numVcpus[MAX_VCPUS], n_vcpus; 706 unsigned int jj; 707 const char *vm_name; 708 unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | 709 VIR_CONNECT_LIST_DOMAINS_PERSISTENT; 710 unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; 711 712 if (!global_hypervisor_available) 713 return; 714 715 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 716 if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { 717 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 718 return; 719 } 720 721 /* Returns number of pcpus */ 722 global_n_host_cpus = (unsigned int)node_info.cpus; 723 724 /* Returns number of active domains */ 725 *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, 726 domain_flags); 727 if (*num_vm <= 0) { 728 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); 729 return; 730 } 731 732 for (i = 0; i < *num_vm; i++) { 733 734 /* Get Domain Names */ 735 vm_name = virDomainGetName(domptr[i]); 736 lvm_info[i].vm_name = vm_name; 737 738 /* Get Number of Vcpus */ 739 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); 740 741 /* Get Number of VCpus & VcpuPinInfo */ 742 n_vcpus = virDomainGetVcpuPinInfo(domptr[i], 743 numVcpus[i], global_cpumaps, 744 global_maplen, domain_flag); 745 746 if ((int)n_vcpus > 0) { 747 *num_vcpu = n_vcpus; 748 lvm_info[i].num_cpus = n_vcpus; 749 } 750 751 /* Save pcpu in use by libvirt VMs */ 752 for (ii = 0; ii < n_vcpus; ii++) { 753 for (jj = 0; jj < global_n_host_cpus; jj++) { 754 if (VIR_CPU_USABLE(global_cpumaps, 755 global_maplen, ii, jj) > 0) { 756 lvm_info[i].pcpus[ii] = jj; 757 } 758 } 759 } 760 } 761 } 762 763 int 764 get_info_vm(const char *vm_name, struct vm_info *info) 765 { 766 struct virtual_machine_info *vm_info; 767 unsigned i, channel_num = 0; 768 char mask[RTE_MAX_LCORE]; 769 770 vm_info = find_domain_by_name(vm_name); 771 if (vm_info == NULL) { 772 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 773 return -1; 774 } 775 info->status = CHANNEL_MGR_VM_ACTIVE; 776 if (!virDomainIsActive(vm_info->domainPtr)) 777 info->status = CHANNEL_MGR_VM_INACTIVE; 778 779 rte_spinlock_lock(&(vm_info->config_spinlock)); 780 781 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 782 for (i = 0; i < RTE_MAX_LCORE; i++) { 783 if (mask[i] != 1) 784 continue; 785 info->channels[channel_num].channel_num = i; 786 memcpy(info->channels[channel_num].channel_path, 787 vm_info->channels[i]->channel_path, 788 UNIX_PATH_MAX); 789 info->channels[channel_num].status = 790 vm_info->channels[i]->status; 791 info->channels[channel_num].fd = 792 vm_info->channels[i]->fd; 793 channel_num++; 794 } 795 796 info->allow_query = vm_info->allow_query; 797 info->num_channels = channel_num; 798 info->num_vcpus = vm_info->info.nrVirtCpu; 799 rte_spinlock_unlock(&(vm_info->config_spinlock)); 800 801 memcpy(info->name, vm_info->name, sizeof(vm_info->name)); 802 rte_spinlock_lock(&(vm_info->config_spinlock)); 803 for (i = 0; i < info->num_vcpus; i++) { 804 info->pcpu_map[i] = vm_info->pcpu_map[i]; 805 } 806 rte_spinlock_unlock(&(vm_info->config_spinlock)); 807 return 0; 808 } 809 810 int 811 add_vm(const char *vm_name) 812 { 813 struct virtual_machine_info *new_domain; 814 virDomainPtr dom_ptr; 815 int i; 816 817 if (find_domain_by_name(vm_name) != NULL) { 818 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' " 819 "already exists\n", vm_name); 820 return -1; 821 } 822 823 if (global_vir_conn_ptr == NULL) { 824 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n"); 825 return -1; 826 } 827 dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name); 828 if (dom_ptr == NULL) { 829 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: " 830 "VM '%s' not found\n", vm_name); 831 return -1; 832 } 833 834 new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain), 835 RTE_CACHE_LINE_SIZE); 836 if (new_domain == NULL) { 837 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM " 838 "info\n"); 839 return -1; 840 } 841 new_domain->domainPtr = dom_ptr; 842 if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) { 843 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n"); 844 rte_free(new_domain); 845 return -1; 846 } 847 if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) { 848 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is " 849 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu, 850 RTE_MAX_LCORE); 851 rte_free(new_domain); 852 return -1; 853 } 854 855 for (i = 0; i < RTE_MAX_LCORE; i++) 856 new_domain->pcpu_map[i] = 0; 857 858 if (update_pcpus_mask(new_domain) < 0) { 859 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n"); 860 rte_free(new_domain); 861 return -1; 862 } 863 strncpy(new_domain->name, vm_name, sizeof(new_domain->name)); 864 new_domain->name[sizeof(new_domain->name) - 1] = '\0'; 865 memset(new_domain->channel_mask, 0, RTE_MAX_LCORE); 866 new_domain->num_channels = 0; 867 868 if (!virDomainIsActive(dom_ptr)) 869 new_domain->status = CHANNEL_MGR_VM_INACTIVE; 870 else 871 new_domain->status = CHANNEL_MGR_VM_ACTIVE; 872 873 new_domain->allow_query = 0; 874 rte_spinlock_init(&(new_domain->config_spinlock)); 875 LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info); 876 return 0; 877 } 878 879 int 880 remove_vm(const char *vm_name) 881 { 882 struct virtual_machine_info *vm_info = find_domain_by_name(vm_name); 883 884 if (vm_info == NULL) { 885 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' " 886 "not found\n", vm_name); 887 return -1; 888 } 889 rte_spinlock_lock(&vm_info->config_spinlock); 890 if (vm_info->num_channels != 0) { 891 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are " 892 "%"PRId8" channels still active\n", 893 vm_name, vm_info->num_channels); 894 rte_spinlock_unlock(&vm_info->config_spinlock); 895 return -1; 896 } 897 LIST_REMOVE(vm_info, vms_info); 898 rte_spinlock_unlock(&vm_info->config_spinlock); 899 rte_free(vm_info); 900 return 0; 901 } 902 903 int 904 set_query_status(char *vm_name, 905 bool allow_query) 906 { 907 struct virtual_machine_info *vm_info; 908 909 vm_info = find_domain_by_name(vm_name); 910 if (vm_info == NULL) { 911 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 912 return -1; 913 } 914 rte_spinlock_lock(&(vm_info->config_spinlock)); 915 vm_info->allow_query = allow_query ? 1 : 0; 916 rte_spinlock_unlock(&(vm_info->config_spinlock)); 917 return 0; 918 } 919 920 static void 921 disconnect_hypervisor(void) 922 { 923 if (global_vir_conn_ptr != NULL) { 924 virConnectClose(global_vir_conn_ptr); 925 global_vir_conn_ptr = NULL; 926 } 927 } 928 929 static int 930 connect_hypervisor(const char *path) 931 { 932 if (global_vir_conn_ptr != NULL) { 933 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection " 934 "already established\n", path); 935 return -1; 936 } 937 global_vir_conn_ptr = virConnectOpen(path); 938 if (global_vir_conn_ptr == NULL) { 939 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to " 940 "Hypervisor '%s'\n", path); 941 return -1; 942 } 943 return 0; 944 } 945 int 946 channel_manager_init(const char *path __rte_unused) 947 { 948 virNodeInfo info; 949 950 LIST_INIT(&vm_list_head); 951 if (connect_hypervisor(path) < 0) { 952 global_n_host_cpus = 64; 953 global_hypervisor_available = 0; 954 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n"); 955 } else { 956 global_hypervisor_available = 1; 957 958 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE); 959 960 global_vircpuinfo = rte_zmalloc(NULL, 961 sizeof(*global_vircpuinfo) * 962 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE); 963 if (global_vircpuinfo == NULL) { 964 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n"); 965 goto error; 966 } 967 global_cpumaps = rte_zmalloc(NULL, 968 RTE_MAX_LCORE * global_maplen, 969 RTE_CACHE_LINE_SIZE); 970 if (global_cpumaps == NULL) 971 goto error; 972 973 if (virNodeGetInfo(global_vir_conn_ptr, &info)) { 974 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 975 goto error; 976 } 977 global_n_host_cpus = (unsigned int)info.cpus; 978 } 979 980 981 982 if (global_n_host_cpus > RTE_MAX_LCORE) { 983 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the " 984 "maximum of %u. No cores over %u should be used.\n", 985 global_n_host_cpus, RTE_MAX_LCORE, 986 RTE_MAX_LCORE - 1); 987 global_n_host_cpus = RTE_MAX_LCORE; 988 } 989 990 return 0; 991 error: 992 if (global_hypervisor_available) 993 disconnect_hypervisor(); 994 return -1; 995 } 996 997 void 998 channel_manager_exit(void) 999 { 1000 unsigned i; 1001 char mask[RTE_MAX_LCORE]; 1002 struct virtual_machine_info *vm_info; 1003 1004 LIST_FOREACH(vm_info, &vm_list_head, vms_info) { 1005 1006 rte_spinlock_lock(&(vm_info->config_spinlock)); 1007 1008 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 1009 for (i = 0; i < RTE_MAX_LCORE; i++) { 1010 if (mask[i] != 1) 1011 continue; 1012 remove_channel_from_monitor( 1013 vm_info->channels[i]); 1014 close(vm_info->channels[i]->fd); 1015 rte_free(vm_info->channels[i]); 1016 } 1017 rte_spinlock_unlock(&(vm_info->config_spinlock)); 1018 1019 LIST_REMOVE(vm_info, vms_info); 1020 rte_free(vm_info); 1021 } 1022 1023 if (global_hypervisor_available) { 1024 /* Only needed if hypervisor available */ 1025 rte_free(global_cpumaps); 1026 rte_free(global_vircpuinfo); 1027 disconnect_hypervisor(); 1028 } 1029 } 1030