1# -*- mode: ruby -*- 2# vi: set ft=ruby : 3 4require 'open3' 5def get_box_type(distro, force_distro) 6 spdk_distro = 'spdk/' + distro 7 localboxes, stderr, status = Open3.capture3("vagrant box list") 8 return spdk_distro if localboxes.include?(spdk_distro) 9 10 distro_to_type = { 11 'centos7' => 'centos/7', 12 'centos8' => 'centos/8', 13 'ubuntu1604' => 'peru/ubuntu-16.04-server-amd64', 14 'ubuntu1804' => 'peru/ubuntu-18.04-server-amd64', 15 'ubuntu2004' => 'peru/ubuntu-20.04-server-amd64', 16 'fedora33' => 'generic/fedora33', 17 'fedora34' => 'generic/fedora34', 18 'fedora35' => 'generic/fedora35', 19 'arch' => 'generic/arch', 20 'freebsd12' => 'generic/freebsd12', 21 'rocky8' => 'rockylinux/8' 22 } 23 abort("Invalid argument! #{distro}") unless distro_to_type.key?(distro) || force_distro 24 25 return distro_to_type[distro] ? distro_to_type[distro] : distro 26end 27 28def setup_proxy(config,distro) 29 return unless ENV['http_proxy'] 30 31 if Vagrant.has_plugin?("vagrant-proxyconf") 32 config.proxy.http = ENV['http_proxy'] 33 config.proxy.https = ENV['https_proxy'] 34 config.proxy.no_proxy = "localhost,127.0.0.1" 35 end 36 37 # Proxyconf does not seem to support FreeBSD boxes or at least it's 38 # docs do not mention that. Set up proxy configuration manually. 39 if distro.include?("freebsd") 40 $freebsd_proxy = <<-SCRIPT 41 sudo -s 42 echo "export http_proxy=#{ENV['http_proxy']}" >> /etc/profile 43 echo "export https_proxy=#{ENV['http_proxy']}" >> /etc/profile 44 echo "pkg_env: {http_proxy: #{ENV['http_proxy']}}" > /usr/local/etc/pkg.conf 45 chown root:wheel /usr/local/etc/pkg.conf 46 chmod 644 /usr/local/etc/pkg.conf 47 SCRIPT 48 config.vm.provision "shell", inline: $freebsd_proxy 49 end 50end 51 52def copy_gitconfig(config) 53 src_path = '~/.gitconfig' 54 return unless File.file?(File.expand_path(src_path)) 55 56 config.vm.provision "file", source: src_path, destination: ".gitconfig" 57end 58 59def copy_tsocks(config) 60 tsocks_file = 'tsocks.conf' 61 tsocks_file_path = '/etc/' + tsocks_file 62 63 return unless File.file?(tsocks_file_path) 64 65 $tsocks_copy_cmd = <<-SCRIPT 66 sudo -s 67 mv -f "#{tsocks_file}" "#{tsocks_file_path}" 68 chown root "#{tsocks_file_path}" 69 chmod 644 "#{tsocks_file_path}" 70 SCRIPT 71 72 config.vm.provision "file", source: tsocks_file_path, destination: tsocks_file 73 config.vm.provision "shell", inline: $tsocks_copy_cmd 74end 75 76def copy_vagrant_tools(config,files_sync_backend) 77 src_path = '~/vagrant_tools' 78 return unless File.directory?(File.expand_path(src_path)) 79 80 config.vm.synced_folder src_path, "/home/vagrant/tools", files_sync_backend 81end 82 83def copy_spdk_dir(config, files_sync_backend) 84 return unless ENV['COPY_SPDK_DIR'] == "1" 85 return unless ENV['SPDK_DIR'] 86 87 config.vm.synced_folder ENV['SPDK_DIR'], '/home/vagrant/spdk_repo/spdk', files_sync_backend 88end 89 90def copy_spdk_artifacts(config, plugins_sync_backend) 91 return unless ENV['COPY_SPDK_ARTIFACTS'] == "1" 92 93 vagrantfile_dir=(ENV['VAGRANTFILE_DIR'] || "none") 94 config.vm.synced_folder "#{vagrantfile_dir}/output", "/home/vagrant/spdk_repo/output", plugins_sync_backend 95end 96 97def make_spdk_local_copy_of_nfs(config,distro) 98 user_group = 'vagrant:vagrant' 99 100 spdk_path = '/home/vagrant/spdk_repo/spdk' 101 spdk_tmp_path = '/tmp/spdk' 102 $spdk_repo_cmd = <<-SCRIPT 103 sudo -s 104 cp -R '#{spdk_path}' '#{spdk_tmp_path}' 105 umount '#{spdk_path}' && rm -rf '#{spdk_path}' 106 mv '#{spdk_tmp_path}' '#{spdk_path}' 107 chown -R #{user_group} '#{spdk_path}' 108 SCRIPT 109 110 config.vm.provision "shell", inline: $spdk_repo_cmd 111end 112 113def get_nvme_disk(disk, index) 114 if ENV['NVME_FILE'] 115 nvme_file = ENV['NVME_FILE'].split(',') 116 nvme_disk = nvme_file[index] 117 else 118 nvme_disk = '/var/lib/libvirt/images/nvme_disk.img' 119 end 120 121 unless nvme_disk == "none" || File.exist?(nvme_disk) 122 puts 'If run with libvirt provider please execute create_nvme_img.sh' 123 end 124 125 return nvme_disk 126end 127 128def setup_nvme_disk(libvirt, disk, index) 129 nvme_disk_id = disk + '-' + index.to_s 130 nvme_disk = get_nvme_disk(disk, index) 131 132 nvme_namespaces=(ENV['NVME_DISKS_NAMESPACES'] || "").split(',') 133 nvme_cmbs=(ENV['NVME_CMB'] || "").split(',') 134 nvme_pmrs=(ENV['NVME_PMR'] || "").split(',') 135 nvme_zns=(ENV['NVME_ZNS'] || "").split(',') 136 137 namespace_disks = [] 138 pmr_cmdline = "" 139 nvme_controller = "" 140 141 # Define controller 142 nvme_controller = "nvme,id=#{nvme_disk_id},serial=1234#{index}" 143 144 # Gather all drives - each namespace requires separate drive 145 if nvme_namespaces[index].nil? 146 namespace_disks = namespace_disks + nvme_disk.split() 147 elsif !nvme_namespaces[index].nil? && !nvme_namespaces[index].match(/^[0-9]+$/) 148 namespace_disks = namespace_disks + nvme_disk.split() + nvme_namespaces[index].split(':') 149 elsif !nvme_namespaces[index].nil? && nvme_namespaces[index].match(/^[0-9]+$/) 150 # Compatibility with spdk-5.0.0 fork 151 libvirt.qemuargs :value => "-drive" 152 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 153 if nvme_namespaces[index] == "1" 154 nvme_controller <<",drive=#{nvme_disk_id}" 155 else 156 nvme_controller <<",namespaces=#{nvme_namespaces[index]},drive=#{nvme_disk_id}" 157 end 158 end 159 160 if !nvme_cmbs[index].nil? && nvme_cmbs[index] != "" 161 # Fix the size of the buffer to 128M 162 nvme_controller << ",cmb_size_mb=128" 163 end 164 165 if !nvme_pmrs[index].nil? && nvme_pmrs[index] != "" 166 pmr_path, pmr_size = nvme_pmrs[index].split(':') 167 if pmr_size.nil? 168 pmr_size = "16M" 169 end 170 nvme_controller << ",pmrdev=pmr#{index}" 171 pmr_cmdline = "memory-backend-file,id=pmr#{index},share=on,mem-path=#{pmr_path},size=#{pmr_size}" 172 end 173 174 libvirt.qemuargs :value => "-device" 175 libvirt.qemuargs :value => nvme_controller 176 177 if pmr_cmdline != "" 178 libvirt.qemuargs :value => "-object" 179 libvirt.qemuargs :value => pmr_cmdline 180 end 181 182 # Define all namespaces 183 namespace_disks.each_with_index { |disk, nsid| 184 if disk == "none" 185 next 186 end 187 zoned = nvme_zns[index].nil? ? "false" : "true" 188 libvirt.qemuargs :value => "-drive" 189 libvirt.qemuargs :value => "format=raw,file=#{disk},if=none,id=#{nvme_disk_id}-drive#{nsid}" 190 libvirt.qemuargs :value => "-device" 191 libvirt.qemuargs :value => "nvme-ns,drive=#{nvme_disk_id}-drive#{nsid},bus=#{nvme_disk_id},nsid=#{nsid + 1},zoned=#{zoned},logical_block_size=4096,physical_block_size=4096" 192 } 193 194end 195 196def setup_ssh(config) 197 config.ssh.forward_agent = true 198 config.ssh.forward_x11 = true 199 if ENV['VAGRANT_PASSWORD_AUTH'] == "1" 200 config.ssh.username = "vagrant" 201 config.ssh.password = "vagrant" 202 config.ssh.private_key_path = nil 203 end 204end 205 206def deploy_test_vm(config, distro, plugins_sync_backend) 207 return unless ENV['DEPLOY_TEST_VM'] == "1" 208 return unless ENV['COPY_SPDK_DIR'] == "1" 209 return unless ENV['SPDK_DIR'] 210 211 # use http proxy if available 212 setup_proxy(config, distro) 213 214 # Copy the tsocks configuration file for use when installing some spdk test pool dependencies 215 copy_tsocks(config) 216 217 # freebsd boxes in order to have spdk sources synced from 218 # host properly will use NFS with "ro" option enabled to prevent changes 219 # on host filesystem. 220 # To make sources usable in the guest VM we need to unmount them and use 221 # local copy. 222 make_spdk_local_copy_of_nfs(config,distro) if plugins_sync_backend[:type] == :nfs 223 224 config.vm.provision "shell" do |setup| 225 setup.inline = "/home/vagrant/spdk_repo/spdk/test/common/config/vm_setup.sh" 226 setup.privileged = false 227 setup.args = ["-u", "-i"] 228 end 229end 230 231def setup_virtualbox(config, vmcpu, vmram) 232 config.vm.provider "virtualbox" do |vb| 233 vb.customize ["modifyvm", :id, "--ioapic", "on"] 234 vb.memory = vmram 235 vb.cpus = vmcpu 236 237 nvme_disk=(ENV['NVME_FILE'] || "nvme_disk.img") 238 unless File.exist? (nvme_disk) 239 vb.customize ["createhd", "--filename", nvme_disk, "--variant", "Fixed", "--size", "1024"] 240 vb.customize ["storagectl", :id, "--name", "nvme", "--add", "pcie", "--controller", "NVMe", "--portcount", "1", "--bootable", "off"] 241 vb.customize ["storageattach", :id, "--storagectl", "nvme", "--type", "hdd", "--medium", nvme_disk, "--port", "0"] 242 end 243 244 #support for the SSE4.x instruction is required in some versions of VB. 245 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] 246 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] 247 end 248end 249 250def setup_libvirt(config, vmcpu, vmram, distro) 251 emulated_nvme_types=(ENV['NVME_DISKS_TYPE'] || "nvme").split(',') 252 253 config.vm.provider "libvirt" do |libvirt, override| 254 libvirt.random_hostname = "1" 255 libvirt.driver = "kvm" 256 libvirt.graphics_type = "vnc" 257 libvirt.memory = vmram 258 libvirt.cpus = vmcpu 259 libvirt.video_type = "cirrus" 260 261 if (distro.include?("freebsd")) 262 # generic/freebsd boxes need to be explicitly run with SCSI bus, 263 # otherwise boot process fails on mounting the disk 264 libvirt.disk_bus = "scsi" 265 elsif (distro.include?("arch")) 266 # Run generic/arch boxes explicitly with IDE bus, 267 # otherwise boot process fails on mounting the disk 268 libvirt.disk_bus = "ide" 269 else 270 libvirt.disk_bus = "virtio" 271 end 272 273 if ENV['SPDK_QEMU_EMULATOR'] 274 libvirt.emulator_path = ENV['SPDK_QEMU_EMULATOR'] 275 libvirt.machine_type = "pc" 276 end 277 278 # we put nvme_disk inside default pool to eliminate libvirt/SELinux Permissions Problems 279 # and to be able to run vagrant from user $HOME directory 280 281 # Loop to create all emulated disks set 282 emulated_nvme_types.each_with_index { |disk, index| 283 setup_nvme_disk(libvirt, disk, index) 284 } 285 286 # Add network interface for openstack tests 287 if ENV['SPDK_OPENSTACK_NETWORK'] == "1" 288 libvirt.qemuargs :value => "-device" 289 libvirt.qemuargs :value => "virtio-net,netdev=openstack.0" 290 libvirt.qemuargs :value => "-netdev" 291 libvirt.qemuargs :value => "user,id=openstack.0" 292 end 293 294 if ENV['VAGRANT_HUGE_MEM'] == "1" 295 libvirt.memorybacking :hugepages 296 end 297 298 # Optional field if we want use other storage pools than default 299 # libvirt.storage_pool_name = "vm" 300 end 301end 302 303################################################################################################# 304# Pick the right distro and bootstrap, default is fedora33 305distro = (ENV['SPDK_VAGRANT_DISTRO'] || "fedora33") 306provider = (ENV['SPDK_VAGRANT_PROVIDER'] || "virtualbox") 307 308# Get all variables for creating vm 309vmcpu = (ENV['SPDK_VAGRANT_VMCPU'] || 2) 310vmram = (ENV['SPDK_VAGRANT_VMRAM'] || 4096) 311 312# generic/freebsd boxes do not work properly with vagrant-proxyconf and 313# have issues installing rsync and sshfs for syncing files. NFS is 314# pre-installed, so use it. 315# generic/fedora boxes on the other hand have problems running NFS 316# service so use sshfs+rsync combo instead. 317force_distro = ENV['FORCE_DISTRO'] == "true" ? true : false 318 319distro_to_use = get_box_type(distro, force_distro) 320if (distro_to_use.include?("generic/freebsd")) 321 files_sync_backend = {type: :nfs, nfs_udp: false, mount_options: ['ro']} 322 plugins_sync_backend = {type: :nfs, nfs_udp: false} 323else 324 # Remove --copy-links from default rsync cmdline since we do want to sync 325 # actual symlinks as well. Also, since copy is made between host and its 326 # local VM we don't need to worry about saturating the local link so skip 327 # the compression to speed up the whole transfer. 328 files_sync_backend = {type: "rsync", rsync__auto: false, rsync__args: ["--archive", "--verbose", "--delete"]} 329 plugins_sync_backend = {type: :sshfs} 330end 331 332Vagrant.configure(2) do |config| 333 config.vm.box = distro_to_use 334 config.vm.box_check_update = false 335 config.vm.synced_folder '.', '/vagrant', disabled: true 336 if ENV['VAGRANT_BOX_VERSION'] 337 config.vm.box_version = ENV['VAGRANT_BOX_VERSION'] 338 end 339 340 # Copy in the .gitconfig if it exists 341 copy_gitconfig(config) 342 343 # Copy in the user's tools if they exists 344 copy_vagrant_tools(config,files_sync_backend) 345 346 # rsync the spdk directory if provision hasn't happened yet 347 # Warning: rsync does not work with freebsd boxes, so this step is disabled 348 copy_spdk_dir(config, files_sync_backend) 349 350 # rsync artifacts from build 351 copy_spdk_artifacts(config, plugins_sync_backend) 352 353 # Setup SSH 354 setup_ssh(config) 355 356 # Virtualbox configuration 357 setup_virtualbox(config,vmcpu,vmram) 358 359 # This setup was Tested on Fedora 27 360 # libvirt configuration need modern Qemu(tested on 2.10) & vagrant-libvirt in version 0.0.39+ 361 # There are few limitation for SElinux - The file added outside libvirt must have proper SE ACL policy or setenforce 0 362 setup_libvirt(config,vmcpu,vmram,distro) 363 364 # provision the vm with all of the necessary spdk dependencies for running the autorun.sh tests 365 deploy_test_vm(config, distro, plugins_sync_backend) 366end 367