1# -*- mode: ruby -*- 2# vi: set ft=ruby : 3 4require 'open3' 5def get_box_type(distro, force_distro) 6 spdk_distro = 'spdk/' + distro 7 localboxes, stderr, status = Open3.capture3("vagrant box list") 8 return spdk_distro if localboxes.include?(spdk_distro) 9 10 distro_to_type = { 11 'centos7' => 'centos/7', 12 'ubuntu2004' => 'peru/ubuntu-20.04-server-amd64', 13 'ubuntu2204' => 'generic/ubuntu2204', 14 'fedora37' => 'generic/fedora37', 15 'fedora38' => 'generic/fedora38', 16 'arch' => 'generic/arch', 17 'freebsd12' => 'generic/freebsd12', 18 'freebsd13' => 'generic/freebsd13', 19 'rocky8' => 'rockylinux/8', 20 'rocky9' => 'rockylinux/9' 21 } 22 abort("Invalid argument! #{distro}") unless distro_to_type.key?(distro) || force_distro 23 24 return distro_to_type[distro] ? distro_to_type[distro] : distro 25end 26 27def setup_proxy(config,distro) 28 return unless ENV['http_proxy'] 29 30 if Vagrant.has_plugin?("vagrant-proxyconf") 31 config.proxy.http = ENV['http_proxy'] 32 config.proxy.https = ENV['https_proxy'] 33 config.proxy.no_proxy = "localhost,127.0.0.1" 34 end 35 36 # Proxyconf does not seem to support FreeBSD boxes or at least it's 37 # docs do not mention that. Set up proxy configuration manually. 38 if distro.include?("freebsd") 39 $freebsd_proxy = <<-SCRIPT 40 sudo -s 41 echo "export http_proxy=#{ENV['http_proxy']}" >> /etc/profile 42 echo "export https_proxy=#{ENV['http_proxy']}" >> /etc/profile 43 echo "pkg_env: {http_proxy: #{ENV['http_proxy']}}" > /usr/local/etc/pkg.conf 44 chown root:wheel /usr/local/etc/pkg.conf 45 chmod 644 /usr/local/etc/pkg.conf 46 SCRIPT 47 config.vm.provision "shell", inline: $freebsd_proxy 48 end 49end 50 51def copy_gitconfig(config) 52 src_path = '~/.gitconfig' 53 return unless File.file?(File.expand_path(src_path)) 54 55 config.vm.provision "file", source: src_path, destination: ".gitconfig" 56end 57 58def copy_tsocks(config) 59 tsocks_file = 'tsocks.conf' 60 tsocks_file_path = '/etc/' + tsocks_file 61 62 return unless File.file?(tsocks_file_path) 63 64 $tsocks_copy_cmd = <<-SCRIPT 65 sudo -s 66 mv -f "#{tsocks_file}" "#{tsocks_file_path}" 67 chown root "#{tsocks_file_path}" 68 chmod 644 "#{tsocks_file_path}" 69 SCRIPT 70 71 config.vm.provision "file", source: tsocks_file_path, destination: tsocks_file 72 config.vm.provision "shell", inline: $tsocks_copy_cmd 73end 74 75def copy_vagrant_tools(config,files_sync_backend) 76 src_path = '~/vagrant_tools' 77 return unless File.directory?(File.expand_path(src_path)) 78 79 config.vm.synced_folder src_path, "/home/vagrant/tools", files_sync_backend 80end 81 82def copy_sources_dirs(config, files_sync_backend) 83 return unless ENV['COPY_SPDK_DIR'] == "1" 84 return unless ENV['SPDK_DIR'] 85 86 repo_prefix = '/home/vagrant/spdk_repo' 87 config.vm.synced_folder ENV['SPDK_DIR'], "#{repo_prefix}/spdk", files_sync_backend 88 89 # Optional directories 90 for dir in ['spdk-abi', 'dpdk'] 91 src_path = "#{ENV['SPDK_DIR']}/../#{dir}" 92 next unless File.directory?(File.expand_path(src_path)) 93 94 config.vm.synced_folder src_path, "#{repo_prefix}/#{dir}", files_sync_backend 95 end 96end 97 98def copy_spdk_artifacts(config, plugins_sync_backend) 99 return unless ENV['COPY_SPDK_ARTIFACTS'] == "1" 100 101 vagrantfile_dir=(ENV['VAGRANTFILE_DIR'] || "none") 102 config.vm.synced_folder "#{vagrantfile_dir}/output", "/home/vagrant/spdk_repo/output", plugins_sync_backend 103end 104 105def make_spdk_local_copy_of_nfs(config,distro) 106 user_group = 'vagrant:vagrant' 107 108 spdk_path = '/home/vagrant/spdk_repo/spdk' 109 spdk_tmp_path = '/tmp/spdk' 110 $spdk_repo_cmd = <<-SCRIPT 111 sudo -s 112 cp -R '#{spdk_path}' '#{spdk_tmp_path}' 113 umount '#{spdk_path}' && rm -rf '#{spdk_path}' 114 mv '#{spdk_tmp_path}' '#{spdk_path}' 115 chown -R #{user_group} '#{spdk_path}' 116 SCRIPT 117 118 config.vm.provision "shell", inline: $spdk_repo_cmd 119end 120 121def get_nvme_disk(disk, index) 122 if ENV['NVME_FILE'] 123 nvme_file = ENV['NVME_FILE'].split(',') 124 nvme_disk = nvme_file[index] 125 else 126 nvme_disk = '/var/lib/libvirt/images/nvme_disk.img' 127 end 128 129 unless nvme_disk == "none" || File.exist?(nvme_disk) 130 puts 'If run with libvirt provider please execute create_nvme_img.sh' 131 end 132 133 return nvme_disk 134end 135 136def setup_nvme_disk(libvirt, disk, index) 137 nvme_disk_id = disk + '-' + index.to_s 138 nvme_disk = get_nvme_disk(disk, index) 139 140 nvme_namespaces=(ENV['NVME_DISKS_NAMESPACES'] || "").split(',') 141 nvme_cmbs=(ENV['NVME_CMB'] || "").split(',') 142 nvme_pmrs=(ENV['NVME_PMR'] || "").split(',') 143 nvme_zns=(ENV['NVME_ZNS'] || "").split(',') 144 nvme_ms=(ENV['NVME_MS'] || "").split(',') 145 nvme_fdp=(ENV['NVME_FDP'] || "").split(',') 146 147 namespace_disks = [] 148 pmr_cmdline = "" 149 nvme_controller = "" 150 fdp_subsys = "" 151 fdp_subsys_id = "" 152 ms = "" 153 154 addr = 0x10 + index 155 156 # Define controller 157 nvme_controller = "nvme,id=#{nvme_disk_id},serial=1234#{index},addr=0x#{addr.to_s(16)}" 158 159 # For the FDP, we need to hook our nvme into a dedicated subsystem 160 if !nvme_fdp[index].nil? && nvme_fdp[index] != "" 161 fdp_subsys_id = "fdp-subsys#{index}" 162 fdp = nvme_fdp[index].split(':')[0..3] 163 fdp_ruhs = (nvme_fdp[index].split(':')[4..]) # fdp.ruhs per ns 164 165 # Put some defaults in place if needed 166 fdp_enable = "#{fdp[0] != nil && fdp[0] != '' ? fdp[0] : 'off'}" 167 fdp_runs = "#{fdp[1] != nil && fdp[1] != '' ? fdp[1] : '96M'}" 168 fdp_nrg = "#{fdp[2] != nil && fdp[2] != '' ? fdp[2] : 2}" 169 fdp_nruh = "#{fdp[3] != nil && fdp[3] != '' ? fdp[3] : 8}" 170 171 fdp_subsys = "nvme-subsys,id=#{fdp_subsys_id},fdp=#{fdp_enable}" 172 fdp_subsys << ",fdp.runs=#{fdp_runs},fdp.nrg=#{fdp_nrg},fdp.nruh=#{fdp_nruh}" 173 174 nvme_controller << ",subsys=#{fdp_subsys_id}" 175 176 libvirt.qemuargs :value => "-device" 177 libvirt.qemuargs :value => fdp_subsys 178 end 179 # Gather all drives - each namespace requires separate drive 180 if nvme_namespaces[index].nil? 181 namespace_disks = namespace_disks + nvme_disk.split() 182 elsif !nvme_namespaces[index].nil? && !nvme_namespaces[index].match(/^[0-9]+$/) 183 namespace_disks = namespace_disks + nvme_disk.split() + nvme_namespaces[index].split(':') 184 elsif !nvme_namespaces[index].nil? && nvme_namespaces[index] == "1" 185 libvirt.qemuargs :value => "-drive" 186 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 187 nvme_controller <<",drive=#{nvme_disk_id}" 188 end 189 190 if !nvme_cmbs[index].nil? && nvme_cmbs[index] != "" 191 # Fix the size of the buffer to 128M 192 nvme_controller << ",cmb_size_mb=128" 193 end 194 195 if !nvme_pmrs[index].nil? && nvme_pmrs[index] != "" 196 pmr_path, pmr_size = nvme_pmrs[index].split(':') 197 if pmr_size.nil? 198 pmr_size = "16M" 199 end 200 nvme_controller << ",pmrdev=pmr#{index}" 201 pmr_cmdline = "memory-backend-file,id=pmr#{index},share=on,mem-path=#{pmr_path},size=#{pmr_size}" 202 end 203 204 libvirt.qemuargs :value => "-device" 205 libvirt.qemuargs :value => nvme_controller 206 207 if pmr_cmdline != "" 208 libvirt.qemuargs :value => "-object" 209 libvirt.qemuargs :value => pmr_cmdline 210 end 211 212 # Define all namespaces 213 namespace_disks.each_with_index { |disk, nsid| 214 if disk == "none" 215 next 216 end 217 zoned = nvme_zns[index].nil? ? "false" : "true" 218 if ! nvme_ms[index].nil? && nvme_ms[index] == "true" 219 ms = ",ms=64" 220 end 221 ns = "nvme-ns,drive=#{nvme_disk_id}-drive#{nsid},bus=#{nvme_disk_id},nsid=#{nsid + 1},zoned=#{zoned},logical_block_size=4096,physical_block_size=4096#{ms}" 222 if !fdp_ruhs.nil? && !fdp_ruhs[nsid].nil? && fdp_ruhs[nsid] != "" 223 ns << ",fdp.ruhs=#{fdp_ruhs[nsid]}" 224 end 225 libvirt.qemuargs :value => "-drive" 226 libvirt.qemuargs :value => "format=raw,file=#{disk},if=none,id=#{nvme_disk_id}-drive#{nsid}" 227 libvirt.qemuargs :value => "-device" 228 libvirt.qemuargs :value => ns 229 } 230 231end 232 233def setup_ssh(config) 234 config.ssh.forward_agent = true 235 config.ssh.forward_x11 = true 236 if ENV['VAGRANT_PASSWORD_AUTH'] == "1" 237 config.ssh.username = "vagrant" 238 config.ssh.password = "vagrant" 239 config.ssh.private_key_path = nil 240 end 241end 242 243def deploy_test_vm(config, distro, plugins_sync_backend) 244 return unless ENV['DEPLOY_TEST_VM'] == "1" 245 return unless ENV['COPY_SPDK_DIR'] == "1" 246 return unless ENV['SPDK_DIR'] 247 248 # use http proxy if available 249 setup_proxy(config, distro) 250 251 # Copy the tsocks configuration file for use when installing some spdk test pool dependencies 252 copy_tsocks(config) 253 254 # freebsd boxes in order to have spdk sources synced from 255 # host properly will use NFS with "ro" option enabled to prevent changes 256 # on host filesystem. 257 # To make sources usable in the guest VM we need to unmount them and use 258 # local copy. 259 make_spdk_local_copy_of_nfs(config,distro) if plugins_sync_backend[:type] == :nfs 260 261 config.vm.provision "shell" do |setup| 262 setup.inline = "/home/vagrant/spdk_repo/spdk/test/common/config/autotest_setup.sh" 263 setup.privileged = false 264 setup.args = ["-u", "-i"] 265 end 266end 267 268def setup_virtualbox(config, vmcpu, vmram) 269 config.vm.provider "virtualbox" do |vb| 270 vb.customize ["modifyvm", :id, "--ioapic", "on"] 271 vb.memory = vmram 272 vb.cpus = vmcpu 273 274 nvme_disk=(ENV['NVME_FILE'] || "nvme_disk.img") 275 unless File.exist? (nvme_disk) 276 vb.customize ["createhd", "--filename", nvme_disk, "--variant", "Fixed", "--size", "1024"] 277 vb.customize ["storagectl", :id, "--name", "nvme", "--add", "pcie", "--controller", "NVMe", "--portcount", "1", "--bootable", "off"] 278 vb.customize ["storageattach", :id, "--storagectl", "nvme", "--type", "hdd", "--medium", nvme_disk, "--port", "0"] 279 end 280 281 #support for the SSE4.x instruction is required in some versions of VB. 282 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] 283 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] 284 end 285end 286 287def setup_libvirt(config, vmcpu, vmram, distro) 288 emulated_nvme_types=(ENV['NVME_DISKS_TYPE'] || "nvme").split(',') 289 290 config.vm.provider "libvirt" do |libvirt, override| 291 libvirt.random_hostname = "1" 292 libvirt.driver = "kvm" 293 libvirt.graphics_type = "vnc" 294 libvirt.memory = vmram 295 libvirt.cpus = vmcpu 296 libvirt.video_type = "cirrus" 297 libvirt.nic_model_type = (ENV['NIC_MODEL'] || "virtio") 298 299 if (distro.include?("freebsd")) 300 # generic/freebsd boxes need to be explicitly run with SCSI bus, 301 # otherwise boot process fails on mounting the disk 302 libvirt.disk_bus = "scsi" 303 elsif (distro.include?("arch")) 304 # Run generic/arch boxes explicitly with IDE bus, 305 # otherwise boot process fails on mounting the disk 306 libvirt.disk_bus = "ide" 307 else 308 libvirt.disk_bus = "virtio" 309 end 310 311 if ENV['SPDK_QEMU_EMULATOR'] 312 libvirt.emulator_path = ENV['SPDK_QEMU_EMULATOR'] 313 libvirt.machine_type = "pc" 314 end 315 316 # we put nvme_disk inside default pool to eliminate libvirt/SELinux Permissions Problems 317 # and to be able to run vagrant from user $HOME directory 318 319 # Loop to create all emulated disks set 320 emulated_nvme_types.each_with_index { |disk, index| 321 setup_nvme_disk(libvirt, disk, index) 322 } 323 324 # Add network interface for openstack tests 325 if ENV['SPDK_OPENSTACK_NETWORK'] == "1" 326 libvirt.qemuargs :value => "-device" 327 libvirt.qemuargs :value => "virtio-net,netdev=openstack.0" 328 libvirt.qemuargs :value => "-netdev" 329 libvirt.qemuargs :value => "user,id=openstack.0" 330 end 331 332 if ENV['VAGRANT_HUGE_MEM'] == "1" 333 libvirt.memorybacking :hugepages 334 end 335 336 # Optional field if we want use other storage pools than default 337 # libvirt.storage_pool_name = "vm" 338 end 339end 340 341################################################################################################# 342# Pick the right distro and bootstrap, default is fedora37 343distro = (ENV['SPDK_VAGRANT_DISTRO'] || "fedora37") 344provider = (ENV['SPDK_VAGRANT_PROVIDER'] || "virtualbox") 345 346# Get all variables for creating vm 347vmcpu = (ENV['SPDK_VAGRANT_VMCPU'] || 2) 348vmram = (ENV['SPDK_VAGRANT_VMRAM'] || 4096) 349 350force_distro = ENV['FORCE_DISTRO'] == "true" ? true : false 351 352distro_to_use = get_box_type(distro, force_distro) 353# Remove --copy-links from default rsync cmdline since we do want to sync 354# actual symlinks as well. Also, since copy is made between host and its 355# local VM we don't need to worry about saturating the local link so skip 356# the compression to speed up the whole transfer. 357files_sync_backend = {type: "rsync", rsync__auto: false, rsync__args: ["--archive", "--verbose", "--delete"]} 358 359if ENV['NFS4_BACKEND'] or not Vagrant.has_plugin?("vagrant-sshfs") 360 plugins_sync_backend = {type: :nfs, nfs_udp: false, nfs_version: 4} 361else 362 plugins_sync_backend = {type: :sshfs} 363end 364 365Vagrant.configure(2) do |config| 366 config.vm.box = distro_to_use 367 config.vm.box_check_update = false 368 config.vm.synced_folder '.', '/vagrant', disabled: true 369 if ENV['VAGRANT_BOX_VERSION'] 370 config.vm.box_version = ENV['VAGRANT_BOX_VERSION'] 371 end 372 373 # Copy in the .gitconfig if it exists 374 copy_gitconfig(config) 375 376 # Copy in the user's tools if they exists 377 copy_vagrant_tools(config,files_sync_backend) 378 379 copy_sources_dirs(config, files_sync_backend) 380 381 # rsync artifacts from build 382 copy_spdk_artifacts(config, plugins_sync_backend) 383 384 # Setup SSH 385 setup_ssh(config) 386 387 # Virtualbox configuration 388 setup_virtualbox(config,vmcpu,vmram) 389 390 setup_libvirt(config,vmcpu,vmram,distro) 391 392 # provision the vm with all of the necessary spdk dependencies for running the autorun.sh tests 393 deploy_test_vm(config, distro, plugins_sync_backend) 394end 395 396if ENV['EXTRA_VAGRANTFILES'] 397 loaders = (ENV['EXTRA_VAGRANTFILES'].split(',')) 398 loaders.each { |loader| 399 load loader if File.exists?(loader) 400 } 401end 402