1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <sys/un.h> 8 #include <fcntl.h> 9 #include <unistd.h> 10 #include <inttypes.h> 11 #include <dirent.h> 12 #include <errno.h> 13 14 #include <sys/queue.h> 15 #include <sys/types.h> 16 #include <sys/socket.h> 17 #include <sys/select.h> 18 19 #include <rte_malloc.h> 20 #include <rte_memory.h> 21 #include <rte_mempool.h> 22 #include <rte_log.h> 23 #include <rte_atomic.h> 24 #include <rte_spinlock.h> 25 26 #include <libvirt/libvirt.h> 27 28 #include "channel_manager.h" 29 #include "channel_commands.h" 30 #include "channel_monitor.h" 31 32 33 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1 34 35 #define ITERATIVE_BITMASK_CHECK_64(mask_u64b, i) \ 36 for (i = 0; mask_u64b; mask_u64b &= ~(1ULL << i++)) \ 37 if ((mask_u64b >> i) & 1) \ 38 39 /* Global pointer to libvirt connection */ 40 static virConnectPtr global_vir_conn_ptr; 41 42 static unsigned char *global_cpumaps; 43 static virVcpuInfo *global_vircpuinfo; 44 static size_t global_maplen; 45 46 static unsigned global_n_host_cpus; 47 48 /* 49 * Represents a single Virtual Machine 50 */ 51 struct virtual_machine_info { 52 char name[CHANNEL_MGR_MAX_NAME_LEN]; 53 rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS]; 54 struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS]; 55 uint64_t channel_mask; 56 uint8_t num_channels; 57 enum vm_status status; 58 virDomainPtr domainPtr; 59 virDomainInfo info; 60 rte_spinlock_t config_spinlock; 61 LIST_ENTRY(virtual_machine_info) vms_info; 62 }; 63 64 LIST_HEAD(, virtual_machine_info) vm_list_head; 65 66 static struct virtual_machine_info * 67 find_domain_by_name(const char *name) 68 { 69 struct virtual_machine_info *info; 70 LIST_FOREACH(info, &vm_list_head, vms_info) { 71 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1)) 72 return info; 73 } 74 return NULL; 75 } 76 77 static int 78 update_pcpus_mask(struct virtual_machine_info *vm_info) 79 { 80 virVcpuInfoPtr cpuinfo; 81 unsigned i, j; 82 int n_vcpus; 83 uint64_t mask; 84 85 memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen); 86 87 if (!virDomainIsActive(vm_info->domainPtr)) { 88 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr, 89 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen, 90 VIR_DOMAIN_AFFECT_CONFIG); 91 if (n_vcpus < 0) { 92 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 93 "in-active VM '%s'\n", vm_info->name); 94 return -1; 95 } 96 goto update_pcpus; 97 } 98 99 memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)* 100 CHANNEL_CMDS_MAX_CPUS); 101 102 cpuinfo = global_vircpuinfo; 103 104 n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo, 105 CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen); 106 if (n_vcpus < 0) { 107 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for " 108 "active VM '%s'\n", vm_info->name); 109 return -1; 110 } 111 update_pcpus: 112 if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) { 113 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range " 114 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1); 115 return -1; 116 } 117 if (n_vcpus != vm_info->info.nrVirtCpu) { 118 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s" 119 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu, 120 n_vcpus); 121 vm_info->info.nrVirtCpu = n_vcpus; 122 } 123 for (i = 0; i < vm_info->info.nrVirtCpu; i++) { 124 mask = 0; 125 for (j = 0; j < global_n_host_cpus; j++) { 126 if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) { 127 mask |= 1ULL << j; 128 } 129 } 130 rte_atomic64_set(&vm_info->pcpu_mask[i], mask); 131 } 132 return 0; 133 } 134 135 int 136 set_pcpus_mask(char *vm_name, unsigned vcpu, uint64_t core_mask) 137 { 138 unsigned i = 0; 139 int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG; 140 struct virtual_machine_info *vm_info; 141 uint64_t mask = core_mask; 142 143 if (vcpu >= CHANNEL_CMDS_MAX_CPUS) { 144 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n", 145 vcpu, CHANNEL_CMDS_MAX_CPUS-1); 146 return -1; 147 } 148 149 vm_info = find_domain_by_name(vm_name); 150 if (vm_info == NULL) { 151 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 152 return -1; 153 } 154 155 if (!virDomainIsActive(vm_info->domainPtr)) { 156 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 157 "mask(0x%"PRIx64") for VM '%s', VM is not active\n", 158 vcpu, core_mask, vm_info->name); 159 return -1; 160 } 161 162 if (vcpu >= vm_info->info.nrVirtCpu) { 163 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of " 164 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu); 165 return -1; 166 } 167 memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen); 168 ITERATIVE_BITMASK_CHECK_64(mask, i) { 169 VIR_USE_CPU(global_cpumaps, i); 170 if (i >= global_n_host_cpus) { 171 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available " 172 "number of CPUs(%u)\n", i, global_n_host_cpus); 173 return -1; 174 } 175 } 176 if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps, 177 global_maplen, flags) < 0) { 178 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU " 179 "mask(0x%"PRIx64") for VM '%s'\n", vcpu, core_mask, 180 vm_info->name); 181 return -1; 182 } 183 rte_atomic64_set(&vm_info->pcpu_mask[vcpu], core_mask); 184 return 0; 185 186 } 187 188 int 189 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num) 190 { 191 uint64_t mask = 1ULL << core_num; 192 193 return set_pcpus_mask(vm_name, vcpu, mask); 194 } 195 196 uint64_t 197 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu) 198 { 199 struct virtual_machine_info *vm_info = 200 (struct virtual_machine_info *)chan_info->priv_info; 201 return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]); 202 } 203 204 static inline int 205 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num) 206 { 207 rte_spinlock_lock(&(vm_info->config_spinlock)); 208 if (vm_info->channel_mask & (1ULL << channel_num)) { 209 rte_spinlock_unlock(&(vm_info->config_spinlock)); 210 return 1; 211 } 212 rte_spinlock_unlock(&(vm_info->config_spinlock)); 213 return 0; 214 } 215 216 217 218 static int 219 open_non_blocking_channel(struct channel_info *info) 220 { 221 int ret, flags; 222 struct sockaddr_un sock_addr; 223 fd_set soc_fd_set; 224 struct timeval tv; 225 226 info->fd = socket(AF_UNIX, SOCK_STREAM, 0); 227 if (info->fd == -1) { 228 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n", 229 strerror(errno), 230 info->channel_path); 231 return -1; 232 } 233 sock_addr.sun_family = AF_UNIX; 234 memcpy(&sock_addr.sun_path, info->channel_path, 235 strlen(info->channel_path)+1); 236 237 /* Get current flags */ 238 flags = fcntl(info->fd, F_GETFL, 0); 239 if (flags < 0) { 240 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for" 241 "'%s'\n", strerror(errno), info->channel_path); 242 return 1; 243 } 244 /* Set to Non Blocking */ 245 flags |= O_NONBLOCK; 246 if (fcntl(info->fd, F_SETFL, flags) < 0) { 247 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking " 248 "socket for '%s'\n", strerror(errno), info->channel_path); 249 return -1; 250 } 251 ret = connect(info->fd, (struct sockaddr *)&sock_addr, 252 sizeof(sock_addr)); 253 if (ret < 0) { 254 /* ECONNREFUSED error is given when VM is not active */ 255 if (errno == ECONNREFUSED) { 256 RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not " 257 "activated its endpoint to channel %s\n", 258 info->channel_path); 259 return -1; 260 } 261 /* Wait for tv_sec if in progress */ 262 else if (errno == EINPROGRESS) { 263 tv.tv_sec = 2; 264 tv.tv_usec = 0; 265 FD_ZERO(&soc_fd_set); 266 FD_SET(info->fd, &soc_fd_set); 267 if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) { 268 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel " 269 "'%s'\n", info->channel_path); 270 return -1; 271 } 272 } else { 273 /* Any other error */ 274 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket" 275 " for '%s'\n", strerror(errno), info->channel_path); 276 return -1; 277 } 278 } 279 return 0; 280 } 281 282 static int 283 setup_channel_info(struct virtual_machine_info **vm_info_dptr, 284 struct channel_info **chan_info_dptr, unsigned channel_num) 285 { 286 struct channel_info *chan_info = *chan_info_dptr; 287 struct virtual_machine_info *vm_info = *vm_info_dptr; 288 289 chan_info->channel_num = channel_num; 290 chan_info->priv_info = (void *)vm_info; 291 chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED; 292 if (open_non_blocking_channel(chan_info) < 0) { 293 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: " 294 "'%s' for VM '%s'\n", 295 chan_info->channel_path, vm_info->name); 296 return -1; 297 } 298 if (add_channel_to_monitor(&chan_info) < 0) { 299 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: " 300 "'%s' to epoll ctl for VM '%s'\n", 301 chan_info->channel_path, vm_info->name); 302 return -1; 303 304 } 305 rte_spinlock_lock(&(vm_info->config_spinlock)); 306 vm_info->num_channels++; 307 vm_info->channel_mask |= 1ULL << channel_num; 308 vm_info->channels[channel_num] = chan_info; 309 chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED; 310 rte_spinlock_unlock(&(vm_info->config_spinlock)); 311 return 0; 312 } 313 314 int 315 add_all_channels(const char *vm_name) 316 { 317 DIR *d; 318 struct dirent *dir; 319 struct virtual_machine_info *vm_info; 320 struct channel_info *chan_info; 321 char *token, *remaining, *tail_ptr; 322 char socket_name[PATH_MAX]; 323 unsigned channel_num; 324 int num_channels_enabled = 0; 325 326 /* verify VM exists */ 327 vm_info = find_domain_by_name(vm_name); 328 if (vm_info == NULL) { 329 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found" 330 " during channel discovery\n", vm_name); 331 return 0; 332 } 333 if (!virDomainIsActive(vm_info->domainPtr)) { 334 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 335 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 336 return 0; 337 } 338 d = opendir(CHANNEL_MGR_SOCKET_PATH); 339 if (d == NULL) { 340 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n", 341 CHANNEL_MGR_SOCKET_PATH, strerror(errno)); 342 return -1; 343 } 344 while ((dir = readdir(d)) != NULL) { 345 if (!strncmp(dir->d_name, ".", 1) || 346 !strncmp(dir->d_name, "..", 2)) 347 continue; 348 349 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name); 350 remaining = socket_name; 351 /* Extract vm_name from "<vm_name>.<channel_num>" */ 352 token = strsep(&remaining, "."); 353 if (remaining == NULL) 354 continue; 355 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN)) 356 continue; 357 358 /* remaining should contain only <channel_num> */ 359 errno = 0; 360 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0); 361 if ((errno != 0) || (remaining[0] == '\0') || 362 tail_ptr == NULL || (*tail_ptr != '\0')) { 363 RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name" 364 "'%s' found it should be in the form of " 365 "'<guest_name>.<channel_num>(decimal)'\n", 366 dir->d_name); 367 continue; 368 } 369 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) { 370 RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is " 371 "greater than max allowable: %d, skipping '%s%s'\n", 372 channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1, 373 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 374 continue; 375 } 376 /* if channel has not been added previously */ 377 if (channel_exists(vm_info, channel_num)) 378 continue; 379 380 chan_info = rte_malloc(NULL, sizeof(*chan_info), 381 RTE_CACHE_LINE_SIZE); 382 if (chan_info == NULL) { 383 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 384 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name); 385 continue; 386 } 387 388 snprintf(chan_info->channel_path, 389 sizeof(chan_info->channel_path), "%s%s", 390 CHANNEL_MGR_SOCKET_PATH, dir->d_name); 391 392 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) { 393 rte_free(chan_info); 394 continue; 395 } 396 397 num_channels_enabled++; 398 } 399 closedir(d); 400 return num_channels_enabled; 401 } 402 403 int 404 add_channels(const char *vm_name, unsigned *channel_list, 405 unsigned len_channel_list) 406 { 407 struct virtual_machine_info *vm_info; 408 struct channel_info *chan_info; 409 char socket_path[PATH_MAX]; 410 unsigned i; 411 int num_channels_enabled = 0; 412 413 vm_info = find_domain_by_name(vm_name); 414 if (vm_info == NULL) { 415 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 416 "not found\n", vm_name); 417 return 0; 418 } 419 420 if (!virDomainIsActive(vm_info->domainPtr)) { 421 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name); 422 vm_info->status = CHANNEL_MGR_VM_INACTIVE; 423 return 0; 424 } 425 426 for (i = 0; i < len_channel_list; i++) { 427 428 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) { 429 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range " 430 "0...%d\n", channel_list[i], 431 CHANNEL_CMDS_MAX_VM_CHANNELS-1); 432 continue; 433 } 434 if (channel_exists(vm_info, channel_list[i])) { 435 RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping " 436 "'%s.%u'\n", vm_name, i); 437 continue; 438 } 439 440 snprintf(socket_path, sizeof(socket_path), "%s%s.%u", 441 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 442 errno = 0; 443 if (access(socket_path, F_OK) < 0) { 444 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: " 445 "%s\n", socket_path, strerror(errno)); 446 continue; 447 } 448 chan_info = rte_malloc(NULL, sizeof(*chan_info), 449 RTE_CACHE_LINE_SIZE); 450 if (chan_info == NULL) { 451 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for " 452 "channel '%s'\n", socket_path); 453 continue; 454 } 455 snprintf(chan_info->channel_path, 456 sizeof(chan_info->channel_path), "%s%s.%u", 457 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]); 458 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) { 459 rte_free(chan_info); 460 continue; 461 } 462 num_channels_enabled++; 463 464 } 465 return num_channels_enabled; 466 } 467 468 int 469 remove_channel(struct channel_info **chan_info_dptr) 470 { 471 struct virtual_machine_info *vm_info; 472 struct channel_info *chan_info = *chan_info_dptr; 473 474 close(chan_info->fd); 475 476 vm_info = (struct virtual_machine_info *)chan_info->priv_info; 477 478 rte_spinlock_lock(&(vm_info->config_spinlock)); 479 vm_info->channel_mask &= ~(1ULL << chan_info->channel_num); 480 vm_info->num_channels--; 481 rte_spinlock_unlock(&(vm_info->config_spinlock)); 482 483 rte_free(chan_info); 484 return 0; 485 } 486 487 int 488 set_channel_status_all(const char *vm_name, enum channel_status status) 489 { 490 struct virtual_machine_info *vm_info; 491 unsigned i; 492 uint64_t mask; 493 int num_channels_changed = 0; 494 495 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 496 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 497 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 498 "disabled: Unable to change status for VM '%s'\n", vm_name); 499 } 500 vm_info = find_domain_by_name(vm_name); 501 if (vm_info == NULL) { 502 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' " 503 "not found\n", vm_name); 504 return 0; 505 } 506 507 rte_spinlock_lock(&(vm_info->config_spinlock)); 508 mask = vm_info->channel_mask; 509 ITERATIVE_BITMASK_CHECK_64(mask, i) { 510 vm_info->channels[i]->status = status; 511 num_channels_changed++; 512 } 513 rte_spinlock_unlock(&(vm_info->config_spinlock)); 514 return num_channels_changed; 515 516 } 517 518 int 519 set_channel_status(const char *vm_name, unsigned *channel_list, 520 unsigned len_channel_list, enum channel_status status) 521 { 522 struct virtual_machine_info *vm_info; 523 unsigned i; 524 int num_channels_changed = 0; 525 526 if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED || 527 status == CHANNEL_MGR_CHANNEL_DISABLED)) { 528 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or " 529 "disabled: Unable to change status for VM '%s'\n", vm_name); 530 } 531 vm_info = find_domain_by_name(vm_name); 532 if (vm_info == NULL) { 533 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' " 534 "not found\n", vm_name); 535 return 0; 536 } 537 for (i = 0; i < len_channel_list; i++) { 538 if (channel_exists(vm_info, channel_list[i])) { 539 rte_spinlock_lock(&(vm_info->config_spinlock)); 540 vm_info->channels[channel_list[i]]->status = status; 541 rte_spinlock_unlock(&(vm_info->config_spinlock)); 542 num_channels_changed++; 543 } 544 } 545 return num_channels_changed; 546 } 547 548 void 549 get_all_vm(int *num_vm, int *num_vcpu) 550 { 551 552 virNodeInfo node_info; 553 virDomainPtr *domptr; 554 uint64_t mask; 555 int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus; 556 unsigned int jj; 557 const char *vm_name; 558 unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | 559 VIR_CONNECT_LIST_DOMAINS_PERSISTENT; 560 unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; 561 562 563 memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen); 564 if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { 565 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 566 return; 567 } 568 569 /* Returns number of pcpus */ 570 global_n_host_cpus = (unsigned int)node_info.cpus; 571 572 /* Returns number of active domains */ 573 *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, 574 domain_flags); 575 if (*num_vm <= 0) { 576 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); 577 return; 578 } 579 580 for (i = 0; i < *num_vm; i++) { 581 582 /* Get Domain Names */ 583 vm_name = virDomainGetName(domptr[i]); 584 lvm_info[i].vm_name = vm_name; 585 586 /* Get Number of Vcpus */ 587 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); 588 589 /* Get Number of VCpus & VcpuPinInfo */ 590 n_vcpus = virDomainGetVcpuPinInfo(domptr[i], 591 numVcpus[i], global_cpumaps, 592 global_maplen, domain_flag); 593 594 if ((int)n_vcpus > 0) { 595 *num_vcpu = n_vcpus; 596 lvm_info[i].num_cpus = n_vcpus; 597 } 598 599 /* Save pcpu in use by libvirt VMs */ 600 for (ii = 0; ii < n_vcpus; ii++) { 601 mask = 0; 602 for (jj = 0; jj < global_n_host_cpus; jj++) { 603 if (VIR_CPU_USABLE(global_cpumaps, 604 global_maplen, ii, jj) > 0) { 605 mask |= 1ULL << jj; 606 } 607 } 608 ITERATIVE_BITMASK_CHECK_64(mask, cpu) { 609 lvm_info[i].pcpus[ii] = cpu; 610 } 611 } 612 } 613 } 614 615 int 616 get_info_vm(const char *vm_name, struct vm_info *info) 617 { 618 struct virtual_machine_info *vm_info; 619 unsigned i, channel_num = 0; 620 uint64_t mask; 621 622 vm_info = find_domain_by_name(vm_name); 623 if (vm_info == NULL) { 624 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name); 625 return -1; 626 } 627 info->status = CHANNEL_MGR_VM_ACTIVE; 628 if (!virDomainIsActive(vm_info->domainPtr)) 629 info->status = CHANNEL_MGR_VM_INACTIVE; 630 631 rte_spinlock_lock(&(vm_info->config_spinlock)); 632 633 mask = vm_info->channel_mask; 634 ITERATIVE_BITMASK_CHECK_64(mask, i) { 635 info->channels[channel_num].channel_num = i; 636 memcpy(info->channels[channel_num].channel_path, 637 vm_info->channels[i]->channel_path, UNIX_PATH_MAX); 638 info->channels[channel_num].status = vm_info->channels[i]->status; 639 info->channels[channel_num].fd = vm_info->channels[i]->fd; 640 channel_num++; 641 } 642 643 info->num_channels = channel_num; 644 info->num_vcpus = vm_info->info.nrVirtCpu; 645 rte_spinlock_unlock(&(vm_info->config_spinlock)); 646 647 memcpy(info->name, vm_info->name, sizeof(vm_info->name)); 648 for (i = 0; i < info->num_vcpus; i++) { 649 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]); 650 } 651 return 0; 652 } 653 654 int 655 add_vm(const char *vm_name) 656 { 657 struct virtual_machine_info *new_domain; 658 virDomainPtr dom_ptr; 659 int i; 660 661 if (find_domain_by_name(vm_name) != NULL) { 662 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' " 663 "already exists\n", vm_name); 664 return -1; 665 } 666 667 if (global_vir_conn_ptr == NULL) { 668 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n"); 669 return -1; 670 } 671 dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name); 672 if (dom_ptr == NULL) { 673 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: " 674 "VM '%s' not found\n", vm_name); 675 return -1; 676 } 677 678 new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain), 679 RTE_CACHE_LINE_SIZE); 680 if (new_domain == NULL) { 681 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM " 682 "info\n"); 683 return -1; 684 } 685 new_domain->domainPtr = dom_ptr; 686 if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) { 687 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n"); 688 rte_free(new_domain); 689 return -1; 690 } 691 if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) { 692 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is " 693 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu, 694 CHANNEL_CMDS_MAX_CPUS); 695 rte_free(new_domain); 696 return -1; 697 } 698 699 for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) { 700 rte_atomic64_init(&new_domain->pcpu_mask[i]); 701 } 702 if (update_pcpus_mask(new_domain) < 0) { 703 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n"); 704 rte_free(new_domain); 705 return -1; 706 } 707 strncpy(new_domain->name, vm_name, sizeof(new_domain->name)); 708 new_domain->name[sizeof(new_domain->name) - 1] = '\0'; 709 new_domain->channel_mask = 0; 710 new_domain->num_channels = 0; 711 712 if (!virDomainIsActive(dom_ptr)) 713 new_domain->status = CHANNEL_MGR_VM_INACTIVE; 714 else 715 new_domain->status = CHANNEL_MGR_VM_ACTIVE; 716 717 rte_spinlock_init(&(new_domain->config_spinlock)); 718 LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info); 719 return 0; 720 } 721 722 int 723 remove_vm(const char *vm_name) 724 { 725 struct virtual_machine_info *vm_info = find_domain_by_name(vm_name); 726 727 if (vm_info == NULL) { 728 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' " 729 "not found\n", vm_name); 730 return -1; 731 } 732 rte_spinlock_lock(&vm_info->config_spinlock); 733 if (vm_info->num_channels != 0) { 734 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are " 735 "%"PRId8" channels still active\n", 736 vm_name, vm_info->num_channels); 737 rte_spinlock_unlock(&vm_info->config_spinlock); 738 return -1; 739 } 740 LIST_REMOVE(vm_info, vms_info); 741 rte_spinlock_unlock(&vm_info->config_spinlock); 742 rte_free(vm_info); 743 return 0; 744 } 745 746 static void 747 disconnect_hypervisor(void) 748 { 749 if (global_vir_conn_ptr != NULL) { 750 virConnectClose(global_vir_conn_ptr); 751 global_vir_conn_ptr = NULL; 752 } 753 } 754 755 static int 756 connect_hypervisor(const char *path) 757 { 758 if (global_vir_conn_ptr != NULL) { 759 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection " 760 "already established\n", path); 761 return -1; 762 } 763 global_vir_conn_ptr = virConnectOpen(path); 764 if (global_vir_conn_ptr == NULL) { 765 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to " 766 "Hypervisor '%s'\n", path); 767 return -1; 768 } 769 return 0; 770 } 771 772 int 773 channel_manager_init(const char *path) 774 { 775 virNodeInfo info; 776 777 LIST_INIT(&vm_list_head); 778 if (connect_hypervisor(path) < 0) { 779 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to initialize channel manager\n"); 780 return -1; 781 } 782 783 global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS); 784 785 global_vircpuinfo = rte_zmalloc(NULL, sizeof(*global_vircpuinfo) * 786 CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE); 787 if (global_vircpuinfo == NULL) { 788 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n"); 789 goto error; 790 } 791 global_cpumaps = rte_zmalloc(NULL, CHANNEL_CMDS_MAX_CPUS * global_maplen, 792 RTE_CACHE_LINE_SIZE); 793 if (global_cpumaps == NULL) { 794 goto error; 795 } 796 797 if (virNodeGetInfo(global_vir_conn_ptr, &info)) { 798 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); 799 goto error; 800 } 801 802 global_n_host_cpus = (unsigned)info.cpus; 803 804 if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) { 805 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the " 806 "maximum of %u. No cores over %u should be used.\n", 807 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS, 808 CHANNEL_CMDS_MAX_CPUS - 1); 809 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS; 810 } 811 812 return 0; 813 error: 814 disconnect_hypervisor(); 815 return -1; 816 } 817 818 void 819 channel_manager_exit(void) 820 { 821 unsigned i; 822 uint64_t mask; 823 struct virtual_machine_info *vm_info; 824 825 LIST_FOREACH(vm_info, &vm_list_head, vms_info) { 826 827 rte_spinlock_lock(&(vm_info->config_spinlock)); 828 829 mask = vm_info->channel_mask; 830 ITERATIVE_BITMASK_CHECK_64(mask, i) { 831 remove_channel_from_monitor(vm_info->channels[i]); 832 close(vm_info->channels[i]->fd); 833 rte_free(vm_info->channels[i]); 834 } 835 rte_spinlock_unlock(&(vm_info->config_spinlock)); 836 837 LIST_REMOVE(vm_info, vms_info); 838 rte_free(vm_info); 839 } 840 841 rte_free(global_cpumaps); 842 rte_free(global_vircpuinfo); 843 disconnect_hypervisor(); 844 } 845