1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <fcntl.h> 8 #include <unistd.h> 9 #include <inttypes.h> 10 #include <dirent.h> 11 #include <errno.h> 12 13 #include <sys/queue.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 #include <sys/socket.h> 17 #include <sys/select.h> 18 19 #include <rte_string_fns.h> 20 #include <rte_malloc.h> 21 #include <rte_memory.h> 22 #include <rte_mempool.h> 23 #include <rte_log.h> 24 #include <rte_spinlock.h> 25 26 #include <libvirt/libvirt.h> 27 28 #include "channel_manager.h" 29 #include "channel_monitor.h" 30 #include "power_manager.h" 31 32 33 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1 34 35 struct libvirt_vm_info lvm_info[MAX_CLIENTS]; 36 37 /* Global pointer to libvirt connection */ 38 static virConnectPtr global_vir_conn_ptr; 39 40 static unsigned char *global_cpumaps; 41 static virVcpuInfo *global_vircpuinfo; 42 static size_t global_maplen; 43 44 static unsigned int global_n_host_cpus; 45 static bool global_hypervisor_available; 46 47 /* 48 * Represents a single Virtual Machine 49 */ 50 struct virtual_machine_info { 51 char name[CHANNEL_MGR_MAX_NAME_LEN]; 52 uint16_t pcpu_map[RTE_MAX_LCORE]; 53 struct channel_info *channels[RTE_MAX_LCORE]; 54 char channel_mask[RTE_MAX_LCORE]; 55 uint8_t num_channels; 56 enum vm_status status; 57 virDomainPtr domainPtr; 58 virDomainInfo info; 59 rte_spinlock_t config_spinlock; 60 int allow_query; 61 LIST_ENTRY(virtual_machine_info) vms_info; 62 }; 63 64 LIST_HEAD(, virtual_machine_info) vm_list_head; 65 66 static struct virtual_machine_info * 67 find_domain_by_name(const char *name) 68 { 69 struct virtual_machine_info *info; 70 LIST_FOREACH(info, &vm_list_head, vms_info) { 71 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1)) 72 return info; 73 } 74 return NULL; 75 } 76 77 static int 78 update_pcpus_mask(struct virtual_machine_info *vm_info) 79 { 80 virVcpuInfoPtr cpuinfo; 81 unsigned i, j; 82 int n_vcpus; 83 84 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 85 86 if (!virDomainIsActive(vm_info->domainPtr)) { 87 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr, 88 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen, 89 VIR_DOMAIN_AFFECT_CONFIG); 90 if (n_vcpus < 0) { 91 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 92 "in-active VM '%s'\n", vm_info->name); 93 return -1; 94 } 95 goto update_pcpus; 96 } 97 98 memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)* 99 RTE_MAX_LCORE); 100 101 cpuinfo = global_vircpuinfo; 102 103 n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo, 104 RTE_MAX_LCORE, global_cpumaps, global_maplen); 105 if (n_vcpus < 0) { 106 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 107 "active VM '%s'\n", vm_info->name); 108 return -1; 109 } 110 update_pcpus: 111 if (n_vcpus >= RTE_MAX_LCORE) { 112 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range " 113 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1); 114 return -1; 115 } 116 if (n_vcpus != vm_info->info.nrVirtCpu) { 117 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s" 118 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu, 119 n_vcpus); 120 vm_info->info.nrVirtCpu = n_vcpus; 121 } 122 rte_spinlock_lock(&(vm_info->config_spinlock)); 123 for (i = 0; i < vm_info->info.nrVirtCpu; i++) { 124 for (j = 0; j < global_n_host_cpus; j++) { 125 if (VIR_CPU_USABLE(global_cpumaps, 126 global_maplen, i, j) <= 0) 127 continue; 128 vm_info->pcpu_map[i] = j; 129 } 130 } 131 rte_spinlock_unlock(&(vm_info->config_spinlock)); 132 return 0; 133 } 134 135 int 136 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu) 137 { 138 int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG; 139 struct virtual_machine_info *vm_info; 140 141 if (vcpu >= RTE_MAX_LCORE) { 142 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n", 143 vcpu, RTE_MAX_LCORE-1); 144 return -1; 145 } 146 147 vm_info = find_domain_by_name(vm_name); 148 if (vm_info == NULL) { 149 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 150 return -1; 151 } 152 153 if (!virDomainIsActive(vm_info->domainPtr)) { 154 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 155 " for VM '%s', VM is not active\n", 156 vcpu, vm_info->name); 157 return -1; 158 } 159 160 if (vcpu >= vm_info->info.nrVirtCpu) { 161 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of " 162 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu); 163 return -1; 164 } 165 memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen); 166 167 VIR_USE_CPU(global_cpumaps, pcpu); 168 169 if (pcpu >= global_n_host_cpus) { 170 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available " 171 "number of CPUs(%u)\n", 172 pcpu, global_n_host_cpus); 173 return -1; 174 } 175 176 if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps, 177 global_maplen, flags) < 0) { 178 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 179 " for VM '%s'\n", vcpu, 180 vm_info->name); 181 return -1; 182 } 183 rte_spinlock_lock(&(vm_info->config_spinlock)); 184 vm_info->pcpu_map[vcpu] = pcpu; 185 rte_spinlock_unlock(&(vm_info->config_spinlock)); 186 return 0; 187 } 188 189 uint16_t 190 get_pcpu(struct channel_info *chan_info, unsigned int vcpu) 191 { 192 struct virtual_machine_info *vm_info = 193 (struct virtual_machine_info *)chan_info->priv_info; 194 195 if (global_hypervisor_available && (vm_info != NULL)) { 196 uint16_t pcpu; 197 rte_spinlock_lock(&(vm_info->config_spinlock)); 198 pcpu = vm_info->pcpu_map[vcpu]; 199 rte_spinlock_unlock(&(vm_info->config_spinlock)); 200 return pcpu; 201 } else 202 return 0; 203 } 204 205 static inline int 206 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num) 207 { 208 rte_spinlock_lock(&(vm_info->config_spinlock)); 209 if (vm_info->channel_mask[channel_num] == 1) { 210 rte_spinlock_unlock(&(vm_info->config_spinlock)); 211 return 1; 212 } 213 rte_spinlock_unlock(&(vm_info->config_spinlock)); 214 return 0; 215 } 216 217 218 219 static int 220 open_non_blocking_channel(struct channel_info *info) 221 { 222 int ret, flags; 223 struct sockaddr_un sock_addr; 224 fd_set soc_fd_set; 225 struct timeval tv; 226 227 info->fd = socket(AF_UNIX, SOCK_STREAM, 0); 228 if (info->fd < 0) { 229 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n", 230 strerror(errno), 231 info->channel_path); 232 return -1; 233 } 234 sock_addr.sun_family = AF_UNIX; 235 memcpy(&sock_addr.sun_path, info->channel_path, 236 strlen(info->channel_path)+1); 237 238 /* Get current flags */ 239 flags = fcntl(info->fd, F_GETFL, 0); 240 if (flags < 0) { 241 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 242 "'%s'\n", strerror(errno), info->channel_path); 243 return 1; 244 } 245 /* Set to Non Blocking */ 246 flags |= O_NONBLOCK; 247 if (fcntl(info->fd, F_SETFL, flags) < 0) { 248 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking " 249 "socket for '%s'\n", strerror(errno), info->channel_path); 250 return -1; 251 } 252 ret = connect(info->fd, (struct sockaddr *)&sock_addr, 253 sizeof(sock_addr)); 254 if (ret < 0) { 255 /* ECONNREFUSED error is given when VM is not active */ 256 if (errno == ECONNREFUSED) { 257 RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not " 258 "activated its endpoint to channel %s\n", 259 info->channel_path); 260 return -1; 261 } 262 /* Wait for tv_sec if in progress */ 263 else if (errno == EINPROGRESS) { 264 tv.tv_sec = 2; 265 tv.tv_usec = 0; 266 FD_ZERO(&soc_fd_set); 267 FD_SET(info->fd, &soc_fd_set); 268 if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) { 269 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel " 270 "'%s'\n", info->channel_path); 271 return -1; 272 } 273 } else { 274 /* Any other error */ 275 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket" 276 " for '%s'\n", strerror(errno), info->channel_path); 277 return -1; 278 } 279 } 280 return 0; 281 } 282 283 static int 284 open_host_channel(struct channel_info *info) 285 { 286 int flags; 287 288 info->fd = open(info->channel_path, O_RDWR | O_RSYNC); 289 if (info->fd < 0) { 290 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n", 291 strerror(errno), 292 info->channel_path); 293 return -1; 294 } 295 296 /* Get current flags */ 297 flags = fcntl(info->fd, F_GETFL, 0); 298 if (flags < 0) { 299 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 300 "'%s'\n", strerror(errno), info->channel_path); 301 return 1; 302 } 303 /* Set to Non Blocking */ 304 flags |= O_NONBLOCK; 305 if (fcntl(info->fd, F_SETFL, flags) < 0) { 306 RTE_LOG(WARNING, CHANNEL_MANAGER, 307 "Error(%s) setting non-blocking " 308 "socket for '%s'\n", 309 strerror(errno), info->channel_path); 310 return -1; 311 } 312 return 0; 313 } 314 315 static int 316 setup_channel_info(struct virtual_machine_info **vm_info_dptr, 317 struct channel_info **chan_info_dptr, unsigned channel_num) 318 { 319 struct channel_info *chan_info = *chan_info_dptr; 320 struct virtual_machine_info *vm_info = *vm_info_dptr; 321 322 chan_info->channel_num = channel_num; 323 chan_info->priv_info = (void *)vm_info; 324 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 325 chan_info->type = CHANNEL_TYPE_BINARY; 326 if (open_non_blocking_channel(chan_info) < 0) { 327 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: " 328 "'%s' for VM '%s'\n", 329 chan_info->channel_path, vm_info->name); 330 return -1; 331 } 332 if (add_channel_to_monitor(&chan_info) < 0) { 333 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 334 "'%s' to epoll ctl for VM '%s'\n", 335 chan_info->channel_path, vm_info->name); 336 return -1; 337 338 } 339 rte_spinlock_lock(&(vm_info->config_spinlock)); 340 vm_info->num_channels++; 341 vm_info->channel_mask[channel_num] = 1; 342 vm_info->channels[channel_num] = chan_info; 343 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 344 rte_spinlock_unlock(&(vm_info->config_spinlock)); 345 return 0; 346 } 347 348 static int 349 fifo_path(char *dst, unsigned int len, unsigned int id) 350 { 351 int cnt; 352 353 cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH, 354 CHANNEL_MGR_FIFO_PATTERN_NAME, id); 355 356 if ((cnt < 0) || (cnt > (int)len - 1)) { 357 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper " 358 "string for fifo path\n"); 359 360 return -1; 361 } 362 363 return 0; 364 } 365 366 static int 367 setup_host_channel_info(struct channel_info **chan_info_dptr, 368 unsigned int channel_num) 369 { 370 struct channel_info *chan_info = *chan_info_dptr; 371 372 chan_info->channel_num = channel_num; 373 chan_info->priv_info = (void *)NULL; 374 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 375 chan_info->type = CHANNEL_TYPE_JSON; 376 377 if (open_host_channel(chan_info) < 0) { 378 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: " 379 "'%s'\n", 380 chan_info->channel_path); 381 return -1; 382 } 383 if (add_channel_to_monitor(&chan_info) < 0) { 384 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 385 "'%s' to epoll ctl\n", 386 chan_info->channel_path); 387 return -1; 388 389 } 390 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 391 return 0; 392 } 393 394 int 395 add_all_channels(const char *vm_name) 396 { 397 DIR *d; 398 struct dirent *dir; 399 struct virtual_machine_info *vm_info; 400 struct channel_info *chan_info; 401 char *token, *remaining, *tail_ptr; 402 char socket_name[PATH_MAX]; 403 unsigned channel_num; 404 int num_channels_enabled = 0; 405 406 /* verify VM exists */ 407 vm_info = find_domain_by_name(vm_name); 408 if (vm_info == NULL) { 409 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found" 410 " during channel discovery\n", vm_name); 411 return 0; 412 } 413 if (!virDomainIsActive(vm_info->domainPtr)) { 414 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 415 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 416 return 0; 417 } 418 d = opendir(CHANNEL_MGR_SOCKET_PATH); 419 if (d == NULL) { 420 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n", 421 CHANNEL_MGR_SOCKET_PATH, strerror(errno)); 422 return -1; 423 } 424 while ((dir = readdir(d)) != NULL) { 425 if (!strncmp(dir->d_name, ".", 1) || 426 !strncmp(dir->d_name, "..", 2)) 427 continue; 428 429 strlcpy(socket_name, dir->d_name, sizeof(socket_name)); 430 remaining = socket_name; 431 /* Extract vm_name from "<vm_name>.<channel_num>" */ 432 token = strsep(&remaining, "."); 433 if (remaining == NULL) 434 continue; 435 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN)) 436 continue; 437 438 /* remaining should contain only <channel_num> */ 439 errno = 0; 440 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0); 441 if ((errno != 0) || (remaining[0] == '\0') || 442 tail_ptr == NULL || (*tail_ptr != '\0')) { 443 RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name" 444 "'%s' found it should be in the form of " 445 "'<guest_name>.<channel_num>(decimal)'\n", 446 dir->d_name); 447 continue; 448 } 449 if (channel_num >= RTE_MAX_LCORE) { 450 RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is " 451 "greater than max allowable: %d, skipping '%s%s'\n", 452 channel_num, RTE_MAX_LCORE-1, 453 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 454 continue; 455 } 456 /* if channel has not been added previously */ 457 if (channel_exists(vm_info, channel_num)) 458 continue; 459 460 chan_info = rte_malloc(NULL, sizeof(*chan_info), 461 RTE_CACHE_LINE_SIZE); 462 if (chan_info == NULL) { 463 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 464 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name); 465 continue; 466 } 467 468 if ((size_t)snprintf(chan_info->channel_path, 469 sizeof(chan_info->channel_path), "%s%s", 470 CHANNEL_MGR_SOCKET_PATH, dir->d_name) 471 >= sizeof(chan_info->channel_path)) { 472 RTE_LOG(ERR, CHANNEL_MANAGER, "Pathname too long for channel '%s%s'\n", 473 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 474 rte_free(chan_info); 475 continue; 476 } 477 478 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) { 479 rte_free(chan_info); 480 continue; 481 } 482 483 num_channels_enabled++; 484 } 485 closedir(d); 486 return num_channels_enabled; 487 } 488 489 int 490 add_channels(const char *vm_name, unsigned *channel_list, 491 unsigned len_channel_list) 492 { 493 struct virtual_machine_info *vm_info; 494 struct channel_info *chan_info; 495 char socket_path[PATH_MAX]; 496 unsigned i; 497 int num_channels_enabled = 0; 498 499 vm_info = find_domain_by_name(vm_name); 500 if (vm_info == NULL) { 501 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 502 "not found\n", vm_name); 503 return 0; 504 } 505 506 if (!virDomainIsActive(vm_info->domainPtr)) { 507 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 508 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 509 return 0; 510 } 511 512 for (i = 0; i < len_channel_list; i++) { 513 if (channel_list[i] >= RTE_MAX_LCORE) { 514 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range " 515 "0...%d\n", channel_list[i], 516 RTE_MAX_LCORE-1); 517 continue; 518 } 519 if (channel_exists(vm_info, channel_list[i])) { 520 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping " 521 "'%s.%u'\n", vm_name, i); 522 continue; 523 } 524 525 snprintf(socket_path, sizeof(socket_path), "%s%s.%u", 526 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 527 errno = 0; 528 if (access(socket_path, F_OK) < 0) { 529 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: " 530 "%s\n", socket_path, strerror(errno)); 531 continue; 532 } 533 chan_info = rte_malloc(NULL, sizeof(*chan_info), 534 RTE_CACHE_LINE_SIZE); 535 if (chan_info == NULL) { 536 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 537 "channel '%s'\n", socket_path); 538 continue; 539 } 540 snprintf(chan_info->channel_path, 541 sizeof(chan_info->channel_path), "%s%s.%u", 542 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 543 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) { 544 rte_free(chan_info); 545 continue; 546 } 547 num_channels_enabled++; 548 549 } 550 return num_channels_enabled; 551 } 552 553 int 554 add_host_channels(void) 555 { 556 struct channel_info *chan_info; 557 char socket_path[PATH_MAX]; 558 int num_channels_enabled = 0; 559 int ret; 560 struct core_info *ci; 561 struct channel_info *chan_infos[RTE_MAX_LCORE]; 562 int i; 563 564 for (i = 0; i < RTE_MAX_LCORE; i++) 565 chan_infos[i] = NULL; 566 567 ci = get_core_info(); 568 if (ci == NULL) { 569 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n"); 570 return 0; 571 } 572 573 for (i = 0; i < ci->core_count; i++) { 574 if (rte_lcore_index(i) == -1) 575 continue; 576 577 if (ci->cd[i].global_enabled_cpus == 0) 578 continue; 579 580 ret = fifo_path(socket_path, sizeof(socket_path), i); 581 if (ret < 0) 582 goto error; 583 584 ret = mkfifo(socket_path, 0660); 585 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n", 586 socket_path); 587 if ((errno != EEXIST) && (ret < 0)) { 588 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: " 589 "%s\n", socket_path, strerror(errno)); 590 goto error; 591 } 592 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0); 593 if (chan_info == NULL) { 594 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 595 "channel '%s'\n", socket_path); 596 goto error; 597 } 598 chan_infos[i] = chan_info; 599 strlcpy(chan_info->channel_path, socket_path, 600 sizeof(chan_info->channel_path)); 601 602 if (setup_host_channel_info(&chan_info, i) < 0) { 603 rte_free(chan_info); 604 chan_infos[i] = NULL; 605 goto error; 606 } 607 num_channels_enabled++; 608 } 609 610 return num_channels_enabled; 611 error: 612 /* Clean up the channels opened before we hit an error. */ 613 for (i = 0; i < ci->core_count; i++) { 614 if (chan_infos[i] != NULL) { 615 remove_channel_from_monitor(chan_infos[i]); 616 close(chan_infos[i]->fd); 617 rte_free(chan_infos[i]); 618 } 619 } 620 return 0; 621 } 622 623 int 624 remove_channel(struct channel_info **chan_info_dptr) 625 { 626 struct virtual_machine_info *vm_info; 627 struct channel_info *chan_info = *chan_info_dptr; 628 629 close(chan_info->fd); 630 631 vm_info = (struct virtual_machine_info *)chan_info->priv_info; 632 633 rte_spinlock_lock(&(vm_info->config_spinlock)); 634 vm_info->channel_mask[chan_info->channel_num] = 0; 635 vm_info->num_channels--; 636 rte_spinlock_unlock(&(vm_info->config_spinlock)); 637 638 rte_free(chan_info); 639 return 0; 640 } 641 642 int 643 set_channel_status_all(const char *vm_name, enum channel_status status) 644 { 645 struct virtual_machine_info *vm_info; 646 unsigned i; 647 char mask[RTE_MAX_LCORE]; 648 int num_channels_changed = 0; 649 650 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 651 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 652 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 653 "disabled: Unable to change status for VM '%s'\n", vm_name); 654 } 655 vm_info = find_domain_by_name(vm_name); 656 if (vm_info == NULL) { 657 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' " 658 "not found\n", vm_name); 659 return 0; 660 } 661 662 rte_spinlock_lock(&(vm_info->config_spinlock)); 663 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 664 for (i = 0; i < RTE_MAX_LCORE; i++) { 665 if (mask[i] != 1) 666 continue; 667 vm_info->channels[i]->status = status; 668 num_channels_changed++; 669 } 670 rte_spinlock_unlock(&(vm_info->config_spinlock)); 671 return num_channels_changed; 672 673 } 674 675 int 676 set_channel_status(const char *vm_name, unsigned *channel_list, 677 unsigned len_channel_list, enum channel_status status) 678 { 679 struct virtual_machine_info *vm_info; 680 unsigned i; 681 int num_channels_changed = 0; 682 683 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 684 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 685 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 686 "disabled: Unable to change status for VM '%s'\n", vm_name); 687 } 688 vm_info = find_domain_by_name(vm_name); 689 if (vm_info == NULL) { 690 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 691 "not found\n", vm_name); 692 return 0; 693 } 694 for (i = 0; i < len_channel_list; i++) { 695 if (channel_exists(vm_info, channel_list[i])) { 696 rte_spinlock_lock(&(vm_info->config_spinlock)); 697 vm_info->channels[channel_list[i]]->status = status; 698 rte_spinlock_unlock(&(vm_info->config_spinlock)); 699 num_channels_changed++; 700 } 701 } 702 return num_channels_changed; 703 } 704 705 void 706 get_all_vm(int *num_vm, int *num_vcpu) 707 { 708 709 virNodeInfo node_info; 710 virDomainPtr *domptr; 711 int i, ii, numVcpus[MAX_VCPUS], n_vcpus; 712 unsigned int jj; 713 const char *vm_name; 714 unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | 715 VIR_CONNECT_LIST_DOMAINS_PERSISTENT; 716 unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; 717 718 if (!global_hypervisor_available) 719 return; 720 721 memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen); 722 if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { 723 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 724 return; 725 } 726 727 /* Returns number of pcpus */ 728 global_n_host_cpus = (unsigned int)node_info.cpus; 729 730 /* Returns number of active domains */ 731 *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, 732 domain_flags); 733 if (*num_vm <= 0) { 734 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); 735 return; 736 } 737 738 for (i = 0; i < *num_vm; i++) { 739 740 /* Get Domain Names */ 741 vm_name = virDomainGetName(domptr[i]); 742 lvm_info[i].vm_name = vm_name; 743 744 /* Get Number of Vcpus */ 745 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); 746 747 /* Get Number of VCpus & VcpuPinInfo */ 748 n_vcpus = virDomainGetVcpuPinInfo(domptr[i], 749 numVcpus[i], global_cpumaps, 750 global_maplen, domain_flag); 751 752 if ((int)n_vcpus > 0) { 753 *num_vcpu = n_vcpus; 754 lvm_info[i].num_cpus = n_vcpus; 755 } 756 757 /* Save pcpu in use by libvirt VMs */ 758 for (ii = 0; ii < n_vcpus; ii++) { 759 for (jj = 0; jj < global_n_host_cpus; jj++) { 760 if (VIR_CPU_USABLE(global_cpumaps, 761 global_maplen, ii, jj) > 0) { 762 lvm_info[i].pcpus[ii] = jj; 763 } 764 } 765 } 766 } 767 } 768 769 int 770 get_info_vm(const char *vm_name, struct vm_info *info) 771 { 772 struct virtual_machine_info *vm_info; 773 unsigned i, channel_num = 0; 774 char mask[RTE_MAX_LCORE]; 775 776 vm_info = find_domain_by_name(vm_name); 777 if (vm_info == NULL) { 778 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 779 return -1; 780 } 781 info->status = CHANNEL_MGR_VM_ACTIVE; 782 if (!virDomainIsActive(vm_info->domainPtr)) 783 info->status = CHANNEL_MGR_VM_INACTIVE; 784 785 rte_spinlock_lock(&(vm_info->config_spinlock)); 786 787 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 788 for (i = 0; i < RTE_MAX_LCORE; i++) { 789 if (mask[i] != 1) 790 continue; 791 info->channels[channel_num].channel_num = i; 792 memcpy(info->channels[channel_num].channel_path, 793 vm_info->channels[i]->channel_path, 794 UNIX_PATH_MAX); 795 info->channels[channel_num].status = 796 vm_info->channels[i]->status; 797 info->channels[channel_num].fd = 798 vm_info->channels[i]->fd; 799 channel_num++; 800 } 801 802 info->allow_query = vm_info->allow_query; 803 info->num_channels = channel_num; 804 info->num_vcpus = vm_info->info.nrVirtCpu; 805 rte_spinlock_unlock(&(vm_info->config_spinlock)); 806 807 memcpy(info->name, vm_info->name, sizeof(vm_info->name)); 808 rte_spinlock_lock(&(vm_info->config_spinlock)); 809 for (i = 0; i < info->num_vcpus; i++) { 810 info->pcpu_map[i] = vm_info->pcpu_map[i]; 811 } 812 rte_spinlock_unlock(&(vm_info->config_spinlock)); 813 return 0; 814 } 815 816 int 817 add_vm(const char *vm_name) 818 { 819 struct virtual_machine_info *new_domain; 820 virDomainPtr dom_ptr; 821 int i; 822 823 if (find_domain_by_name(vm_name) != NULL) { 824 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' " 825 "already exists\n", vm_name); 826 return -1; 827 } 828 829 if (global_vir_conn_ptr == NULL) { 830 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n"); 831 return -1; 832 } 833 dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name); 834 if (dom_ptr == NULL) { 835 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: " 836 "VM '%s' not found\n", vm_name); 837 return -1; 838 } 839 840 new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain), 841 RTE_CACHE_LINE_SIZE); 842 if (new_domain == NULL) { 843 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM " 844 "info\n"); 845 return -1; 846 } 847 new_domain->domainPtr = dom_ptr; 848 if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) { 849 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n"); 850 rte_free(new_domain); 851 return -1; 852 } 853 if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) { 854 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is " 855 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu, 856 RTE_MAX_LCORE); 857 rte_free(new_domain); 858 return -1; 859 } 860 861 for (i = 0; i < RTE_MAX_LCORE; i++) 862 new_domain->pcpu_map[i] = 0; 863 864 if (update_pcpus_mask(new_domain) < 0) { 865 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n"); 866 rte_free(new_domain); 867 return -1; 868 } 869 strncpy(new_domain->name, vm_name, sizeof(new_domain->name)); 870 new_domain->name[sizeof(new_domain->name) - 1] = '\0'; 871 memset(new_domain->channel_mask, 0, RTE_MAX_LCORE); 872 new_domain->num_channels = 0; 873 874 if (!virDomainIsActive(dom_ptr)) 875 new_domain->status = CHANNEL_MGR_VM_INACTIVE; 876 else 877 new_domain->status = CHANNEL_MGR_VM_ACTIVE; 878 879 new_domain->allow_query = 0; 880 rte_spinlock_init(&(new_domain->config_spinlock)); 881 LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info); 882 return 0; 883 } 884 885 int 886 remove_vm(const char *vm_name) 887 { 888 struct virtual_machine_info *vm_info = find_domain_by_name(vm_name); 889 890 if (vm_info == NULL) { 891 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' " 892 "not found\n", vm_name); 893 return -1; 894 } 895 rte_spinlock_lock(&vm_info->config_spinlock); 896 if (vm_info->num_channels != 0) { 897 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are " 898 "%"PRId8" channels still active\n", 899 vm_name, vm_info->num_channels); 900 rte_spinlock_unlock(&vm_info->config_spinlock); 901 return -1; 902 } 903 LIST_REMOVE(vm_info, vms_info); 904 rte_spinlock_unlock(&vm_info->config_spinlock); 905 rte_free(vm_info); 906 return 0; 907 } 908 909 int 910 set_query_status(char *vm_name, 911 bool allow_query) 912 { 913 struct virtual_machine_info *vm_info; 914 915 vm_info = find_domain_by_name(vm_name); 916 if (vm_info == NULL) { 917 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 918 return -1; 919 } 920 rte_spinlock_lock(&(vm_info->config_spinlock)); 921 vm_info->allow_query = allow_query ? 1 : 0; 922 rte_spinlock_unlock(&(vm_info->config_spinlock)); 923 return 0; 924 } 925 926 static void 927 disconnect_hypervisor(void) 928 { 929 if (global_vir_conn_ptr != NULL) { 930 virConnectClose(global_vir_conn_ptr); 931 global_vir_conn_ptr = NULL; 932 } 933 } 934 935 static int 936 connect_hypervisor(const char *path) 937 { 938 if (global_vir_conn_ptr != NULL) { 939 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection " 940 "already established\n", path); 941 return -1; 942 } 943 global_vir_conn_ptr = virConnectOpen(path); 944 if (global_vir_conn_ptr == NULL) { 945 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to " 946 "Hypervisor '%s'\n", path); 947 return -1; 948 } 949 return 0; 950 } 951 int 952 channel_manager_init(const char *path __rte_unused) 953 { 954 virNodeInfo info; 955 956 LIST_INIT(&vm_list_head); 957 if (connect_hypervisor(path) < 0) { 958 global_n_host_cpus = 64; 959 global_hypervisor_available = 0; 960 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n"); 961 } else { 962 global_hypervisor_available = 1; 963 964 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE); 965 966 global_vircpuinfo = rte_zmalloc(NULL, 967 sizeof(*global_vircpuinfo) * 968 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE); 969 if (global_vircpuinfo == NULL) { 970 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n"); 971 goto error; 972 } 973 global_cpumaps = rte_zmalloc(NULL, 974 RTE_MAX_LCORE * global_maplen, 975 RTE_CACHE_LINE_SIZE); 976 if (global_cpumaps == NULL) 977 goto error; 978 979 if (virNodeGetInfo(global_vir_conn_ptr, &info)) { 980 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 981 goto error; 982 } 983 global_n_host_cpus = (unsigned int)info.cpus; 984 } 985 986 987 988 if (global_n_host_cpus > RTE_MAX_LCORE) { 989 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the " 990 "maximum of %u. No cores over %u should be used.\n", 991 global_n_host_cpus, RTE_MAX_LCORE, 992 RTE_MAX_LCORE - 1); 993 global_n_host_cpus = RTE_MAX_LCORE; 994 } 995 996 return 0; 997 error: 998 if (global_hypervisor_available) 999 disconnect_hypervisor(); 1000 return -1; 1001 } 1002 1003 void 1004 channel_manager_exit(void) 1005 { 1006 unsigned i; 1007 char mask[RTE_MAX_LCORE]; 1008 struct virtual_machine_info *vm_info; 1009 1010 LIST_FOREACH(vm_info, &vm_list_head, vms_info) { 1011 1012 rte_spinlock_lock(&(vm_info->config_spinlock)); 1013 1014 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE); 1015 for (i = 0; i < RTE_MAX_LCORE; i++) { 1016 if (mask[i] != 1) 1017 continue; 1018 remove_channel_from_monitor( 1019 vm_info->channels[i]); 1020 close(vm_info->channels[i]->fd); 1021 rte_free(vm_info->channels[i]); 1022 } 1023 rte_spinlock_unlock(&(vm_info->config_spinlock)); 1024 1025 LIST_REMOVE(vm_info, vms_info); 1026 rte_free(vm_info); 1027 } 1028 1029 if (global_hypervisor_available) { 1030 /* Only needed if hypervisor available */ 1031 rte_free(global_cpumaps); 1032 rte_free(global_vircpuinfo); 1033 disconnect_hypervisor(); 1034 } 1035 } 1036