1# -*- mode: ruby -*- 2# vi: set ft=ruby : 3 4require 'open3' 5def get_box_type(distro) 6 spdk_distro = 'spdk/' + distro 7 localboxes, stderr, status = Open3.capture3("vagrant box list") 8 return spdk_distro if localboxes.include?(spdk_distro) 9 10 distro_to_type = { 11 'centos7' => 'centos/7', 12 'centos8' => 'centos/8', 13 'ubuntu1604' => 'peru/ubuntu-16.04-server-amd64', 14 'ubuntu1804' => 'peru/ubuntu-18.04-server-amd64', 15 'ubuntu2004' => 'peru/ubuntu-20.04-server-amd64', 16 'fedora31' => 'generic/fedora31', 17 'fedora32' => 'generic/fedora32', 18 'fedora33' => 'generic/fedora33', 19 'arch' => 'generic/arch', 20 'freebsd12' => 'generic/freebsd12', 21 } 22 abort("Invalid argument! #{distro}") unless distro_to_type.key?(distro) 23 24 return distro_to_type[distro] 25end 26 27def setup_proxy(config,distro) 28 return unless ENV['http_proxy'] 29 30 if Vagrant.has_plugin?("vagrant-proxyconf") 31 config.proxy.http = ENV['http_proxy'] 32 config.proxy.https = ENV['https_proxy'] 33 config.proxy.no_proxy = "localhost,127.0.0.1" 34 end 35 36 # Proxyconf does not seem to support FreeBSD boxes or at least it's 37 # docs do not mention that. Set up proxy configuration manually. 38 if distro.include?("freebsd") 39 $freebsd_proxy = <<-SCRIPT 40 sudo -s 41 echo "export http_proxy=#{ENV['http_proxy']}" >> /etc/profile 42 echo "export https_proxy=#{ENV['http_proxy']}" >> /etc/profile 43 echo "pkg_env: {http_proxy: #{ENV['http_proxy']}}" > /usr/local/etc/pkg.conf 44 chown root:wheel /usr/local/etc/pkg.conf 45 chmod 644 /usr/local/etc/pkg.conf 46 SCRIPT 47 config.vm.provision "shell", inline: $freebsd_proxy 48 end 49end 50 51def copy_gitconfig(config) 52 src_path = '~/.gitconfig' 53 return unless File.file?(File.expand_path(src_path)) 54 55 config.vm.provision "file", source: src_path, destination: ".gitconfig" 56end 57 58def copy_tsocks(config) 59 tsocks_file = 'tsocks.conf' 60 tsocks_file_path = '/etc/' + tsocks_file 61 62 return unless File.file?(tsocks_file_path) 63 64 $tsocks_copy_cmd = <<-SCRIPT 65 sudo -s 66 mv -f "#{tsocks_file}" "#{tsocks_file_path}" 67 chown root "#{tsocks_file_path}" 68 chmod 644 "#{tsocks_file_path}" 69 SCRIPT 70 71 config.vm.provision "file", source: tsocks_file_path, destination: tsocks_file 72 config.vm.provision "shell", inline: $tsocks_copy_cmd 73end 74 75def copy_vagrant_tools(config,files_sync_backend) 76 src_path = '~/vagrant_tools' 77 return unless File.directory?(File.expand_path(src_path)) 78 79 config.vm.synced_folder src_path, "/home/vagrant/tools", files_sync_backend 80end 81 82def copy_spdk_dir(config, files_sync_backend) 83 return unless ENV['COPY_SPDK_DIR'] == "1" 84 return unless ENV['SPDK_DIR'] 85 86 config.vm.synced_folder ENV['SPDK_DIR'], '/home/vagrant/spdk_repo/spdk', files_sync_backend 87end 88 89def copy_spdk_artifacts(config, plugins_sync_backend) 90 return unless ENV['COPY_SPDK_ARTIFACTS'] == "1" 91 92 vagrantfile_dir=(ENV['VAGRANTFILE_DIR'] || "none") 93 config.vm.synced_folder "#{vagrantfile_dir}/output", "/home/vagrant/spdk_repo/output", plugins_sync_backend 94end 95 96def make_spdk_local_copy_of_nfs(config,distro) 97 user_group = 'vagrant:vagrant' 98 99 spdk_path = '/home/vagrant/spdk_repo/spdk' 100 spdk_tmp_path = '/tmp/spdk' 101 $spdk_repo_cmd = <<-SCRIPT 102 sudo -s 103 cp -R '#{spdk_path}' '#{spdk_tmp_path}' 104 umount '#{spdk_path}' && rm -rf '#{spdk_path}' 105 mv '#{spdk_tmp_path}' '#{spdk_path}' 106 chown -R #{user_group} '#{spdk_path}' 107 SCRIPT 108 109 config.vm.provision "shell", inline: $spdk_repo_cmd 110end 111 112def get_nvme_disk(disk, index) 113 if ENV['NVME_FILE'] 114 nvme_file = ENV['NVME_FILE'].split(',') 115 nvme_disk = nvme_file[index] 116 else 117 nvme_disk = '/var/lib/libvirt/images/nvme_disk.img' 118 end 119 120 unless File.exist? (nvme_disk) 121 puts 'If run with libvirt provider please execute create_nvme_img.sh' 122 end 123 124 return nvme_disk 125end 126 127def setup_nvme_disk(libvirt, disk, index) 128 nvme_disk_id = disk + '-' + index.to_s 129 nvme_disk = get_nvme_disk(disk, index) 130 131 nvme_namespaces=(ENV['NVME_DISKS_NAMESPACES'] || "").split(',') 132 nvme_cmbs=(ENV['NVME_CMB'] || "").split(',') 133 nvme_pmrs=(ENV['NVME_PMR'] || "").split(',') 134 135 libvirt.qemuargs :value => "-drive" 136 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 137 libvirt.qemuargs :value => "-device" 138 nvme_drive = "nvme,drive=#{nvme_disk_id},serial=1234#{index}" 139 if !nvme_namespaces[index].nil? && nvme_namespaces[index] != "1" 140 nvme_drive << ",namespaces=#{nvme_namespaces[index]}" 141 end 142 143 cmb_set = "false" 144 pmr_cmdline = 145 146 if !nvme_cmbs[index].nil? && nvme_cmbs[index] == "true" 147 # Fix the size of the buffer to 128M 148 nvme_drive << ",cmb_size_mb=128" 149 cmb_set = "true" 150 end 151 if !nvme_pmrs[index].nil? 152 if cmb_set == "true" 153 abort("CMB and PMR are mutually exclusive, aborting (#{nvme_disk})") 154 end 155 pmr_path, pmr_size = nvme_pmrs[index].split(':') 156 if !File.exist?(pmr_path) 157 abort("#{pmr_path} does not exist, aborting") 158 end 159 if pmr_size.nil? 160 pmr_size = "16M" 161 end 162 nvme_drive << ",pmrdev=pmr#{index}" 163 pmr_cmdline = "memory-backend-file,id=pmr#{index},share=on,mem-path=#{pmr_path},size=#{pmr_size}" 164 end 165 libvirt.qemuargs :value => nvme_drive 166 if !pmr_cmdline.nil? 167 libvirt.qemuargs :value => "-object" 168 libvirt.qemuargs :value => pmr_cmdline 169 end 170end 171 172def setup_ocssd_disk(libvirt, disk, index) 173 nvme_disk_id = disk + '-' + index.to_s 174 nvme_disk = get_nvme_disk(disk, index) 175 176 libvirt.qemuargs :value => "-drive" 177 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 178 libvirt.qemuargs :value => "-device" 179 # create ocssd drive with special parameters 180 # lba_index=4 it is LBA namespace format, 4 means that block size is 4K and have 64B metadata 181 # lnum_lun, lnum_pln, lpgs_per_blk, lsecs_per_pg, lblks_per_pln this are parameters describing the device geometry 182 # we need to multiply these parameters by ourselves to have backend file minimal size: 183 # in our case: 4K * 8 * 2 * 1536 * 2 * 45 = 8640 MB 184 libvirt.qemuargs :value => "nvme,drive=#{nvme_disk_id},serial=deadbeef,oacs=0,namespaces=1,lver=2,lba_index=4,mdts=10,lnum_lun=8,lnum_pln=2,lpgs_per_blk=1536,lsecs_per_pg=2,lblks_per_pln=45,metadata=#{nvme_disk}_ocssd_md,nsdatafile=#{nvme_disk}_ocssd_blknvme.ns,laer_thread_sleep=3000,stride=4" 185end 186 187def setup_ssh(config) 188 config.ssh.forward_agent = true 189 config.ssh.forward_x11 = true 190 if ENV['VAGRANT_PASSWORD_AUTH'] == "1" 191 config.ssh.username = "vagrant" 192 config.ssh.password = "vagrant" 193 config.ssh.private_key_path = nil 194 end 195end 196 197def deploy_test_vm(config, distro, plugins_sync_backend) 198 return unless ENV['DEPLOY_TEST_VM'] == "1" 199 return unless ENV['COPY_SPDK_DIR'] == "1" 200 return unless ENV['SPDK_DIR'] 201 202 # use http proxy if avaiable 203 setup_proxy(config, distro) 204 205 # Copy the tsocks configuration file for use when installing some spdk test pool dependencies 206 copy_tsocks(config) 207 208 # freebsd boxes in order to have spdk sources synced from 209 # host properly will use NFS with "ro" option enabled to prevent changes 210 # on host filesystem. 211 # To make sources usable in the guest VM we need to unmount them and use 212 # local copy. 213 make_spdk_local_copy_of_nfs(config,distro) if plugins_sync_backend[:type] == :nfs 214 215 config.vm.provision "shell" do |setup| 216 setup.inline = "/home/vagrant/spdk_repo/spdk/test/common/config/vm_setup.sh" 217 setup.privileged = false 218 setup.args = ["-u", "-i"] 219 end 220end 221 222def setup_virtualbox(config, vmcpu, vmram) 223 config.vm.provider "virtualbox" do |vb| 224 vb.customize ["modifyvm", :id, "--ioapic", "on"] 225 vb.memory = vmram 226 vb.cpus = vmcpu 227 228 nvme_disk=(ENV['NVME_FILE'] || "nvme_disk.img") 229 unless File.exist? (nvme_disk) 230 vb.customize ["createhd", "--filename", nvme_disk, "--variant", "Fixed", "--size", "1024"] 231 vb.customize ["storagectl", :id, "--name", "nvme", "--add", "pcie", "--controller", "NVMe", "--portcount", "1", "--bootable", "off"] 232 vb.customize ["storageattach", :id, "--storagectl", "nvme", "--type", "hdd", "--medium", nvme_disk, "--port", "0"] 233 end 234 235 #support for the SSE4.x instruction is required in some versions of VB. 236 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] 237 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] 238 end 239end 240 241def setup_libvirt(config, vmcpu, vmram, distro) 242 emulated_nvme_types=(ENV['NVME_DISKS_TYPE'] || "nvme").split(',') 243 244 config.vm.provider "libvirt" do |libvirt, override| 245 libvirt.random_hostname = "1" 246 libvirt.driver = "kvm" 247 libvirt.graphics_type = "vnc" 248 libvirt.memory = vmram 249 libvirt.cpus = vmcpu 250 libvirt.video_type = "cirrus" 251 252 if (distro.include?("freebsd")) 253 # generic/freebsd boxes need to be explicitly run with SCSI bus, 254 # otherwise boot process fails on mounting the disk 255 libvirt.disk_bus = "scsi" 256 elsif (distro.include?("arch")) 257 # Run generic/arch boxes explicitly with IDE bus, 258 # otherwise boot process fails on mounting the disk 259 libvirt.disk_bus = "ide" 260 else 261 libvirt.disk_bus = "virtio" 262 end 263 264 if ENV['SPDK_QEMU_EMULATOR'] 265 libvirt.emulator_path = ENV['SPDK_QEMU_EMULATOR'] 266 libvirt.machine_type = "pc" 267 end 268 269 # we put nvme_disk inside default pool to eliminate libvirt/SELinux Permissions Problems 270 # and to be able to run vagrant from user $HOME directory 271 272 # Loop to create all emulated disks set 273 emulated_nvme_types.each_with_index { |disk, index| 274 if disk == "nvme" 275 setup_nvme_disk(libvirt, disk, index) 276 elsif disk == "ocssd" 277 setup_ocssd_disk(libvirt, disk, index) 278 end 279 } 280 281 if ENV['VAGRANT_HUGE_MEM'] == "1" 282 libvirt.memorybacking :hugepages 283 end 284 285 # Optional field if we want use other storage pools than default 286 # libvirt.storage_pool_name = "vm" 287 end 288end 289 290################################################################################################# 291# Pick the right distro and bootstrap, default is fedora31 292distro = (ENV['SPDK_VAGRANT_DISTRO'] || "fedora31") 293provider = (ENV['SPDK_VAGRANT_PROVIDER'] || "virtualbox") 294 295# Get all variables for creating vm 296vmcpu = (ENV['SPDK_VAGRANT_VMCPU'] || 2) 297vmram = (ENV['SPDK_VAGRANT_VMRAM'] || 4096) 298openstack_network = (ENV['SPDK_OPENSTACK_NETWORK'] || false) 299 300# generic/freebsd boxes do not work properly with vagrant-proxyconf and 301# have issues installing rsync and sshfs for syncing files. NFS is 302# pre-installed, so use it. 303# generic/fedora boxes on the other hand have problems running NFS 304# service so use sshfs+rsync combo instead. 305if (get_box_type(distro).include?("generic/freebsd")) 306 files_sync_backend = {type: :nfs, nfs_udp: false, mount_options: ['ro']} 307 plugins_sync_backend = {type: :nfs, nfs_udp: false} 308else 309 # Remove --copy-links from default rsync cmdline since we do want to sync 310 # actual symlinks as well. Also, since copy is made between host and its 311 # local VM we don't need to worry about saturating the local link so skip 312 # the compression to speed up the whole transfer. 313 files_sync_backend = {type: "rsync", rsync__auto: false, rsync__args: ["--archive", "--verbose", "--delete"]} 314 plugins_sync_backend = {type: :sshfs} 315end 316 317Vagrant.configure(2) do |config| 318 config.vm.box = get_box_type(distro) 319 config.vm.box_check_update = false 320 config.vm.synced_folder '.', '/vagrant', disabled: true 321 322 # Add network interface for openstack tests 323 if openstack_network == "1" 324 config.vm.network "private_network", ip: "10.0.2.15" 325 end 326 # Copy in the .gitconfig if it exists 327 copy_gitconfig(config) 328 329 # Copy in the user's tools if they exists 330 copy_vagrant_tools(config,files_sync_backend) 331 332 # rsync the spdk directory if provision hasn't happened yet 333 # Warning: rsync does not work with freebsd boxes, so this step is disabled 334 copy_spdk_dir(config, files_sync_backend) 335 336 # rsync artifacts from build 337 copy_spdk_artifacts(config, plugins_sync_backend) 338 339 # Setup SSH 340 setup_ssh(config) 341 342 # Virtualbox configuration 343 setup_virtualbox(config,vmcpu,vmram) 344 345 # This setup was Tested on Fedora 27 346 # libvirt configuration need modern Qemu(tested on 2.10) & vagrant-libvirt in version 0.0.39+ 347 # There are few limitation for SElinux - The file added outside libvirt must have proper SE ACL policy or setenforce 0 348 setup_libvirt(config,vmcpu,vmram,distro) 349 350 # provision the vm with all of the necessary spdk dependencies for running the autorun.sh tests 351 deploy_test_vm(config, distro, plugins_sync_backend) 352end 353