1# -*- mode: ruby -*- 2# vi: set ft=ruby : 3 4require 'open3' 5def get_box_type(distro, force_distro) 6 spdk_distro = 'spdk/' + distro 7 localboxes, stderr, status = Open3.capture3("vagrant box list") 8 return spdk_distro if localboxes.include?(spdk_distro) 9 10 distro_to_type = { 11 'centos7' => 'centos/7', 12 'centos8' => 'centos/8', 13 'ubuntu1604' => 'peru/ubuntu-16.04-server-amd64', 14 'ubuntu1804' => 'peru/ubuntu-18.04-server-amd64', 15 'ubuntu2004' => 'peru/ubuntu-20.04-server-amd64', 16 'ubuntu2204' => 'generic/ubuntu2204', 17 'fedora33' => 'generic/fedora33', 18 'fedora34' => 'generic/fedora34', 19 'fedora35' => 'generic/fedora35', 20 'fedora36' => 'generic/fedora36', 21 'arch' => 'generic/arch', 22 'freebsd12' => 'generic/freebsd12', 23 'rocky8' => 'rockylinux/8' 24 } 25 abort("Invalid argument! #{distro}") unless distro_to_type.key?(distro) || force_distro 26 27 return distro_to_type[distro] ? distro_to_type[distro] : distro 28end 29 30def setup_proxy(config,distro) 31 return unless ENV['http_proxy'] 32 33 if Vagrant.has_plugin?("vagrant-proxyconf") 34 config.proxy.http = ENV['http_proxy'] 35 config.proxy.https = ENV['https_proxy'] 36 config.proxy.no_proxy = "localhost,127.0.0.1" 37 end 38 39 # Proxyconf does not seem to support FreeBSD boxes or at least it's 40 # docs do not mention that. Set up proxy configuration manually. 41 if distro.include?("freebsd") 42 $freebsd_proxy = <<-SCRIPT 43 sudo -s 44 echo "export http_proxy=#{ENV['http_proxy']}" >> /etc/profile 45 echo "export https_proxy=#{ENV['http_proxy']}" >> /etc/profile 46 echo "pkg_env: {http_proxy: #{ENV['http_proxy']}}" > /usr/local/etc/pkg.conf 47 chown root:wheel /usr/local/etc/pkg.conf 48 chmod 644 /usr/local/etc/pkg.conf 49 SCRIPT 50 config.vm.provision "shell", inline: $freebsd_proxy 51 end 52end 53 54def copy_gitconfig(config) 55 src_path = '~/.gitconfig' 56 return unless File.file?(File.expand_path(src_path)) 57 58 config.vm.provision "file", source: src_path, destination: ".gitconfig" 59end 60 61def copy_tsocks(config) 62 tsocks_file = 'tsocks.conf' 63 tsocks_file_path = '/etc/' + tsocks_file 64 65 return unless File.file?(tsocks_file_path) 66 67 $tsocks_copy_cmd = <<-SCRIPT 68 sudo -s 69 mv -f "#{tsocks_file}" "#{tsocks_file_path}" 70 chown root "#{tsocks_file_path}" 71 chmod 644 "#{tsocks_file_path}" 72 SCRIPT 73 74 config.vm.provision "file", source: tsocks_file_path, destination: tsocks_file 75 config.vm.provision "shell", inline: $tsocks_copy_cmd 76end 77 78def copy_vagrant_tools(config,files_sync_backend) 79 src_path = '~/vagrant_tools' 80 return unless File.directory?(File.expand_path(src_path)) 81 82 config.vm.synced_folder src_path, "/home/vagrant/tools", files_sync_backend 83end 84 85def copy_spdk_dir(config, files_sync_backend) 86 return unless ENV['COPY_SPDK_DIR'] == "1" 87 return unless ENV['SPDK_DIR'] 88 89 config.vm.synced_folder ENV['SPDK_DIR'], '/home/vagrant/spdk_repo/spdk', files_sync_backend 90end 91 92def copy_spdk_artifacts(config, plugins_sync_backend) 93 return unless ENV['COPY_SPDK_ARTIFACTS'] == "1" 94 95 vagrantfile_dir=(ENV['VAGRANTFILE_DIR'] || "none") 96 config.vm.synced_folder "#{vagrantfile_dir}/output", "/home/vagrant/spdk_repo/output", plugins_sync_backend 97end 98 99def make_spdk_local_copy_of_nfs(config,distro) 100 user_group = 'vagrant:vagrant' 101 102 spdk_path = '/home/vagrant/spdk_repo/spdk' 103 spdk_tmp_path = '/tmp/spdk' 104 $spdk_repo_cmd = <<-SCRIPT 105 sudo -s 106 cp -R '#{spdk_path}' '#{spdk_tmp_path}' 107 umount '#{spdk_path}' && rm -rf '#{spdk_path}' 108 mv '#{spdk_tmp_path}' '#{spdk_path}' 109 chown -R #{user_group} '#{spdk_path}' 110 SCRIPT 111 112 config.vm.provision "shell", inline: $spdk_repo_cmd 113end 114 115def get_nvme_disk(disk, index) 116 if ENV['NVME_FILE'] 117 nvme_file = ENV['NVME_FILE'].split(',') 118 nvme_disk = nvme_file[index] 119 else 120 nvme_disk = '/var/lib/libvirt/images/nvme_disk.img' 121 end 122 123 unless nvme_disk == "none" || File.exist?(nvme_disk) 124 puts 'If run with libvirt provider please execute create_nvme_img.sh' 125 end 126 127 return nvme_disk 128end 129 130def setup_nvme_disk(libvirt, disk, index) 131 nvme_disk_id = disk + '-' + index.to_s 132 nvme_disk = get_nvme_disk(disk, index) 133 134 nvme_namespaces=(ENV['NVME_DISKS_NAMESPACES'] || "").split(',') 135 nvme_cmbs=(ENV['NVME_CMB'] || "").split(',') 136 nvme_pmrs=(ENV['NVME_PMR'] || "").split(',') 137 nvme_zns=(ENV['NVME_ZNS'] || "").split(',') 138 139 namespace_disks = [] 140 pmr_cmdline = "" 141 nvme_controller = "" 142 143 # Define controller 144 nvme_controller = "nvme,id=#{nvme_disk_id},serial=1234#{index}" 145 146 # Gather all drives - each namespace requires separate drive 147 if nvme_namespaces[index].nil? 148 namespace_disks = namespace_disks + nvme_disk.split() 149 elsif !nvme_namespaces[index].nil? && !nvme_namespaces[index].match(/^[0-9]+$/) 150 namespace_disks = namespace_disks + nvme_disk.split() + nvme_namespaces[index].split(':') 151 elsif !nvme_namespaces[index].nil? && nvme_namespaces[index].match(/^[0-9]+$/) 152 # Compatibility with spdk-5.0.0 fork 153 libvirt.qemuargs :value => "-drive" 154 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 155 if nvme_namespaces[index] == "1" 156 nvme_controller <<",drive=#{nvme_disk_id}" 157 else 158 nvme_controller <<",namespaces=#{nvme_namespaces[index]},drive=#{nvme_disk_id}" 159 end 160 end 161 162 if !nvme_cmbs[index].nil? && nvme_cmbs[index] != "" 163 # Fix the size of the buffer to 128M 164 nvme_controller << ",cmb_size_mb=128" 165 end 166 167 if !nvme_pmrs[index].nil? && nvme_pmrs[index] != "" 168 pmr_path, pmr_size = nvme_pmrs[index].split(':') 169 if pmr_size.nil? 170 pmr_size = "16M" 171 end 172 nvme_controller << ",pmrdev=pmr#{index}" 173 pmr_cmdline = "memory-backend-file,id=pmr#{index},share=on,mem-path=#{pmr_path},size=#{pmr_size}" 174 end 175 176 libvirt.qemuargs :value => "-device" 177 libvirt.qemuargs :value => nvme_controller 178 179 if pmr_cmdline != "" 180 libvirt.qemuargs :value => "-object" 181 libvirt.qemuargs :value => pmr_cmdline 182 end 183 184 # Define all namespaces 185 namespace_disks.each_with_index { |disk, nsid| 186 if disk == "none" 187 next 188 end 189 zoned = nvme_zns[index].nil? ? "false" : "true" 190 libvirt.qemuargs :value => "-drive" 191 libvirt.qemuargs :value => "format=raw,file=#{disk},if=none,id=#{nvme_disk_id}-drive#{nsid}" 192 libvirt.qemuargs :value => "-device" 193 libvirt.qemuargs :value => "nvme-ns,drive=#{nvme_disk_id}-drive#{nsid},bus=#{nvme_disk_id},nsid=#{nsid + 1},zoned=#{zoned},logical_block_size=4096,physical_block_size=4096" 194 } 195 196end 197 198def setup_ssh(config) 199 config.ssh.forward_agent = true 200 config.ssh.forward_x11 = true 201 if ENV['VAGRANT_PASSWORD_AUTH'] == "1" 202 config.ssh.username = "vagrant" 203 config.ssh.password = "vagrant" 204 config.ssh.private_key_path = nil 205 end 206end 207 208def deploy_test_vm(config, distro, plugins_sync_backend) 209 return unless ENV['DEPLOY_TEST_VM'] == "1" 210 return unless ENV['COPY_SPDK_DIR'] == "1" 211 return unless ENV['SPDK_DIR'] 212 213 # use http proxy if available 214 setup_proxy(config, distro) 215 216 # Copy the tsocks configuration file for use when installing some spdk test pool dependencies 217 copy_tsocks(config) 218 219 # freebsd boxes in order to have spdk sources synced from 220 # host properly will use NFS with "ro" option enabled to prevent changes 221 # on host filesystem. 222 # To make sources usable in the guest VM we need to unmount them and use 223 # local copy. 224 make_spdk_local_copy_of_nfs(config,distro) if plugins_sync_backend[:type] == :nfs 225 226 config.vm.provision "shell" do |setup| 227 setup.inline = "/home/vagrant/spdk_repo/spdk/test/common/config/vm_setup.sh" 228 setup.privileged = false 229 setup.args = ["-u", "-i"] 230 end 231end 232 233def setup_virtualbox(config, vmcpu, vmram) 234 config.vm.provider "virtualbox" do |vb| 235 vb.customize ["modifyvm", :id, "--ioapic", "on"] 236 vb.memory = vmram 237 vb.cpus = vmcpu 238 239 nvme_disk=(ENV['NVME_FILE'] || "nvme_disk.img") 240 unless File.exist? (nvme_disk) 241 vb.customize ["createhd", "--filename", nvme_disk, "--variant", "Fixed", "--size", "1024"] 242 vb.customize ["storagectl", :id, "--name", "nvme", "--add", "pcie", "--controller", "NVMe", "--portcount", "1", "--bootable", "off"] 243 vb.customize ["storageattach", :id, "--storagectl", "nvme", "--type", "hdd", "--medium", nvme_disk, "--port", "0"] 244 end 245 246 #support for the SSE4.x instruction is required in some versions of VB. 247 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] 248 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] 249 end 250end 251 252def setup_libvirt(config, vmcpu, vmram, distro) 253 emulated_nvme_types=(ENV['NVME_DISKS_TYPE'] || "nvme").split(',') 254 255 config.vm.provider "libvirt" do |libvirt, override| 256 libvirt.random_hostname = "1" 257 libvirt.driver = "kvm" 258 libvirt.graphics_type = "vnc" 259 libvirt.memory = vmram 260 libvirt.cpus = vmcpu 261 libvirt.video_type = "cirrus" 262 263 if (distro.include?("freebsd")) 264 # generic/freebsd boxes need to be explicitly run with SCSI bus, 265 # otherwise boot process fails on mounting the disk 266 libvirt.disk_bus = "scsi" 267 elsif (distro.include?("arch")) 268 # Run generic/arch boxes explicitly with IDE bus, 269 # otherwise boot process fails on mounting the disk 270 libvirt.disk_bus = "ide" 271 else 272 libvirt.disk_bus = "virtio" 273 end 274 275 if ENV['SPDK_QEMU_EMULATOR'] 276 libvirt.emulator_path = ENV['SPDK_QEMU_EMULATOR'] 277 libvirt.machine_type = "pc" 278 end 279 280 # we put nvme_disk inside default pool to eliminate libvirt/SELinux Permissions Problems 281 # and to be able to run vagrant from user $HOME directory 282 283 # Loop to create all emulated disks set 284 emulated_nvme_types.each_with_index { |disk, index| 285 setup_nvme_disk(libvirt, disk, index) 286 } 287 288 # Add network interface for openstack tests 289 if ENV['SPDK_OPENSTACK_NETWORK'] == "1" 290 libvirt.qemuargs :value => "-device" 291 libvirt.qemuargs :value => "virtio-net,netdev=openstack.0" 292 libvirt.qemuargs :value => "-netdev" 293 libvirt.qemuargs :value => "user,id=openstack.0" 294 end 295 296 if ENV['VAGRANT_HUGE_MEM'] == "1" 297 libvirt.memorybacking :hugepages 298 end 299 300 # Optional field if we want use other storage pools than default 301 # libvirt.storage_pool_name = "vm" 302 end 303end 304 305################################################################################################# 306# Pick the right distro and bootstrap, default is fedora33 307distro = (ENV['SPDK_VAGRANT_DISTRO'] || "fedora33") 308provider = (ENV['SPDK_VAGRANT_PROVIDER'] || "virtualbox") 309 310# Get all variables for creating vm 311vmcpu = (ENV['SPDK_VAGRANT_VMCPU'] || 2) 312vmram = (ENV['SPDK_VAGRANT_VMRAM'] || 4096) 313 314# generic/freebsd boxes do not work properly with vagrant-proxyconf and 315# have issues installing rsync and sshfs for syncing files. NFS is 316# pre-installed, so use it. 317# generic/fedora boxes on the other hand have problems running NFS 318# service so use sshfs+rsync combo instead. 319force_distro = ENV['FORCE_DISTRO'] == "true" ? true : false 320 321distro_to_use = get_box_type(distro, force_distro) 322if (distro_to_use.include?("generic/freebsd")) 323 files_sync_backend = {type: :nfs, nfs_udp: false, mount_options: ['ro']} 324 plugins_sync_backend = {type: :nfs, nfs_udp: false} 325else 326 # Remove --copy-links from default rsync cmdline since we do want to sync 327 # actual symlinks as well. Also, since copy is made between host and its 328 # local VM we don't need to worry about saturating the local link so skip 329 # the compression to speed up the whole transfer. 330 files_sync_backend = {type: "rsync", rsync__auto: false, rsync__args: ["--archive", "--verbose", "--delete"]} 331 plugins_sync_backend = {type: :sshfs} 332end 333 334Vagrant.configure(2) do |config| 335 config.vm.box = distro_to_use 336 config.vm.box_check_update = false 337 config.vm.synced_folder '.', '/vagrant', disabled: true 338 if ENV['VAGRANT_BOX_VERSION'] 339 config.vm.box_version = ENV['VAGRANT_BOX_VERSION'] 340 end 341 342 # Copy in the .gitconfig if it exists 343 copy_gitconfig(config) 344 345 # Copy in the user's tools if they exists 346 copy_vagrant_tools(config,files_sync_backend) 347 348 # rsync the spdk directory if provision hasn't happened yet 349 # Warning: rsync does not work with freebsd boxes, so this step is disabled 350 copy_spdk_dir(config, files_sync_backend) 351 352 # rsync artifacts from build 353 copy_spdk_artifacts(config, plugins_sync_backend) 354 355 # Setup SSH 356 setup_ssh(config) 357 358 # Virtualbox configuration 359 setup_virtualbox(config,vmcpu,vmram) 360 361 # This setup was Tested on Fedora 27 362 # libvirt configuration need modern Qemu(tested on 2.10) & vagrant-libvirt in version 0.0.39+ 363 # There are few limitation for SElinux - The file added outside libvirt must have proper SE ACL policy or setenforce 0 364 setup_libvirt(config,vmcpu,vmram,distro) 365 366 # provision the vm with all of the necessary spdk dependencies for running the autorun.sh tests 367 deploy_test_vm(config, distro, plugins_sync_backend) 368end 369