1# -*- mode: ruby -*- 2# vi: set ft=ruby : 3 4require 'open3' 5def get_box_type(distro) 6 spdk_distro = 'spdk/' + distro 7 localboxes, stderr, status = Open3.capture3("vagrant box list") 8 return spdk_distro if localboxes.include?(spdk_distro) 9 10 distro_to_type = { 11 'centos7' => 'centos/7', 12 'centos8' => 'centos/8', 13 'ubuntu1604' => 'peru/ubuntu-16.04-server-amd64', 14 'ubuntu1804' => 'peru/ubuntu-18.04-server-amd64', 15 'ubuntu2004' => 'peru/ubuntu-20.04-server-amd64', 16 'fedora31' => 'generic/fedora31', 17 'fedora32' => 'generic/fedora32', 18 'fedora33' => 'generic/fedora33', 19 'arch' => 'generic/arch', 20 'freebsd11' => 'generic/freebsd11', 21 'freebsd12' => 'generic/freebsd12', 22 'clearlinux' => 'AntonioMeireles/ClearLinux' 23 } 24 abort("Invalid argument! #{distro}") unless distro_to_type.key?(distro) 25 26 return distro_to_type[distro] 27end 28 29def setup_proxy(config,distro) 30 return unless ENV['http_proxy'] 31 32 if Vagrant.has_plugin?("vagrant-proxyconf") 33 config.proxy.http = ENV['http_proxy'] 34 config.proxy.https = ENV['https_proxy'] 35 config.proxy.no_proxy = "localhost,127.0.0.1" 36 end 37 38 # Proxyconf does not seem to support FreeBSD boxes or at least it's 39 # docs do not mention that. Set up proxy configuration manually. 40 if distro.include?("freebsd") 41 $freebsd_proxy = <<-SCRIPT 42 sudo -s 43 echo "export http_proxy=#{ENV['http_proxy']}" >> /etc/profile 44 echo "export https_proxy=#{ENV['http_proxy']}" >> /etc/profile 45 echo "pkg_env: {http_proxy: #{ENV['http_proxy']}}" > /usr/local/etc/pkg.conf 46 chown root:wheel /usr/local/etc/pkg.conf 47 chmod 644 /usr/local/etc/pkg.conf 48 SCRIPT 49 config.vm.provision "shell", inline: $freebsd_proxy 50 end 51end 52 53def copy_gitconfig(config) 54 src_path = '~/.gitconfig' 55 return unless File.file?(File.expand_path(src_path)) 56 57 config.vm.provision "file", source: src_path, destination: ".gitconfig" 58end 59 60def copy_tsocks(config) 61 tsocks_file = 'tsocks.conf' 62 tsocks_file_path = '/etc/' + tsocks_file 63 64 return unless File.file?(tsocks_file_path) 65 66 $tsocks_copy_cmd = <<-SCRIPT 67 sudo -s 68 mv -f "#{tsocks_file}" "#{tsocks_file_path}" 69 chown root "#{tsocks_file_path}" 70 chmod 644 "#{tsocks_file_path}" 71 SCRIPT 72 73 config.vm.provision "file", source: tsocks_file_path, destination: tsocks_file 74 config.vm.provision "shell", inline: $tsocks_copy_cmd 75end 76 77def copy_vagrant_tools(config,files_sync_backend) 78 src_path = '~/vagrant_tools' 79 return unless File.directory?(File.expand_path(src_path)) 80 81 config.vm.synced_folder src_path, "/home/vagrant/tools", files_sync_backend 82end 83 84def copy_spdk_dir(config, files_sync_backend) 85 return unless ENV['COPY_SPDK_DIR'] == "1" 86 return unless ENV['SPDK_DIR'] 87 88 config.vm.synced_folder ENV['SPDK_DIR'], '/home/vagrant/spdk_repo/spdk', files_sync_backend 89end 90 91def copy_spdk_artifacts(config, plugins_sync_backend) 92 return unless ENV['COPY_SPDK_ARTIFACTS'] == "1" 93 94 vagrantfile_dir=(ENV['VAGRANTFILE_DIR'] || "none") 95 config.vm.synced_folder "#{vagrantfile_dir}/output", "/home/vagrant/spdk_repo/output", plugins_sync_backend 96end 97 98def make_spdk_local_copy_of_nfs(config,distro) 99 user_group = 'vagrant:vagrant' 100 if distro.include? 'clearlinux' 101 user_group = 'clear:clear' 102 end 103 104 spdk_path = '/home/vagrant/spdk_repo/spdk' 105 spdk_tmp_path = '/tmp/spdk' 106 $spdk_repo_cmd = <<-SCRIPT 107 sudo -s 108 cp -R '#{spdk_path}' '#{spdk_tmp_path}' 109 umount '#{spdk_path}' && rm -rf '#{spdk_path}' 110 mv '#{spdk_tmp_path}' '#{spdk_path}' 111 chown -R #{user_group} '#{spdk_path}' 112 SCRIPT 113 114 config.vm.provision "shell", inline: $spdk_repo_cmd 115end 116 117def clear_cflags(config) 118 $clearcflags_cmd = <<-SCRIPT 119 echo "export CFLAGS=" >> /etc/profile.d/clearcflags.sh 120 echo "export CFFLAGS=" >> /etc/profile.d/clearcflags.sh 121 echo "export CXXFLAGS=" >> /etc/profile.d/clearcflags.sh 122 echo "export FFLAGS=" >> /etc/profile.d/clearcflags.sh 123 echo "export THEANO_FLAGS=" >> /etc/profile.d/clearcflags.sh 124 SCRIPT 125 config.vm.provision "shell", inline: $clearcflags_cmd, run: "always" 126end 127 128def get_nvme_disk(disk, index) 129 if ENV['NVME_FILE'] 130 nvme_file = ENV['NVME_FILE'].split(',') 131 nvme_disk = nvme_file[index] 132 else 133 nvme_disk = '/var/lib/libvirt/images/nvme_disk.img' 134 end 135 136 unless File.exist? (nvme_disk) 137 puts 'If run with libvirt provider please execute create_nvme_img.sh' 138 end 139 140 return nvme_disk 141end 142 143def setup_nvme_disk(libvirt, disk, index) 144 nvme_disk_id = disk + '-' + index.to_s 145 nvme_disk = get_nvme_disk(disk, index) 146 147 nvme_namespaces=(ENV['NVME_DISKS_NAMESPACES'] || "").split(',') 148 nvme_cmbs=(ENV['NVME_CMB'] || "").split(',') 149 150 libvirt.qemuargs :value => "-drive" 151 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 152 libvirt.qemuargs :value => "-device" 153 nvme_drive = "nvme,drive=#{nvme_disk_id},serial=1234#{index}" 154 if !nvme_namespaces[index].nil? && nvme_namespaces[index] != "1" 155 nvme_drive << ",namespaces=#{nvme_namespaces[index]}" 156 end 157 158 if !nvme_cmbs[index].nil? && nvme_cmbs[index] == "true" 159 # Fix the size of the buffer to 128M 160 nvme_drive << ",cmb_size_mb=128" 161 end 162 libvirt.qemuargs :value => nvme_drive 163end 164 165def setup_ocssd_disk(libvirt, disk, index) 166 nvme_disk_id = disk + '-' + index.to_s 167 nvme_disk = get_nvme_disk(disk, index) 168 169 libvirt.qemuargs :value => "-drive" 170 libvirt.qemuargs :value => "format=raw,file=#{nvme_disk},if=none,id=#{nvme_disk_id}" 171 libvirt.qemuargs :value => "-device" 172 # create ocssd drive with special parameters 173 # lba_index=4 it is LBA namespace format, 4 means that block size is 4K and have 64B metadata 174 # lnum_lun, lnum_pln, lpgs_per_blk, lsecs_per_pg, lblks_per_pln this are parameters describing the device geometry 175 # we need to multiply these parameters by ourselves to have backend file minimal size: 176 # in our case: 4K * 8 * 2 * 1536 * 2 * 45 = 8640 MB 177 libvirt.qemuargs :value => "nvme,drive=#{nvme_disk_id},serial=deadbeef,oacs=0,namespaces=1,lver=2,lba_index=4,mdts=10,lnum_lun=8,lnum_pln=2,lpgs_per_blk=1536,lsecs_per_pg=2,lblks_per_pln=45,metadata=#{nvme_disk}_ocssd_md,nsdatafile=#{nvme_disk}_ocssd_blknvme.ns,laer_thread_sleep=3000,stride=4" 178end 179 180def setup_ssh(config) 181 config.ssh.forward_agent = true 182 config.ssh.forward_x11 = true 183 if ENV['VAGRANT_PASSWORD_AUTH'] == "1" 184 config.ssh.username = "vagrant" 185 config.ssh.password = "vagrant" 186 config.ssh.private_key_path = nil 187 end 188end 189 190def setup_vagrant_cachier(config, plugins_sync_backend) 191 if Vagrant.has_plugin?("vagrant-cachier") 192 config.cache.scope = :box 193 config.cache.synced_folder_opts = plugins_sync_backend 194 else 195 puts 'vagrant-cachier caches apt/yum etc to speed subsequent vagrant up' 196 puts 'to enable install vagrant-cachier plugin: ' 197 puts 'vagrant plugin install vagrant-cachier' 198 end 199end 200 201def deploy_test_vm(config) 202 return unless ENV['DEPLOY_TEST_VM'] == "1" 203 return unless ENV['SPDK_DIR'] 204 205 config.vm.provision "shell" do |setup| 206 setup.path = ENV['SPDK_DIR'] + '/test/common/config/vm_setup.sh' 207 setup.privileged = false 208 setup.args = ["-u", "-i"] 209 end 210end 211 212def setup_virtualbox(config, vmcpu, vmram) 213 config.vm.provider "virtualbox" do |vb| 214 vb.customize ["modifyvm", :id, "--ioapic", "on"] 215 vb.memory = vmram 216 vb.cpus = vmcpu 217 218 nvme_disk=(ENV['NVME_FILE'] || "nvme_disk.img") 219 unless File.exist? (nvme_disk) 220 vb.customize ["createhd", "--filename", nvme_disk, "--variant", "Fixed", "--size", "1024"] 221 vb.customize ["storagectl", :id, "--name", "nvme", "--add", "pcie", "--controller", "NVMe", "--portcount", "1", "--bootable", "off"] 222 vb.customize ["storageattach", :id, "--storagectl", "nvme", "--type", "hdd", "--medium", nvme_disk, "--port", "0"] 223 end 224 225 #support for the SSE4.x instruction is required in some versions of VB. 226 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] 227 vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] 228 end 229end 230 231def setup_libvirt(config, vmcpu, vmram, distro) 232 emulated_nvme_types=(ENV['NVME_DISKS_TYPE'] || "nvme").split(',') 233 234 config.vm.provider "libvirt" do |libvirt, override| 235 libvirt.random_hostname = "1" 236 libvirt.driver = "kvm" 237 libvirt.graphics_type = "vnc" 238 libvirt.memory = vmram 239 libvirt.cpus = vmcpu 240 libvirt.video_type = "cirrus" 241 242 if (distro.include?("freebsd")) 243 # generic/freebsd boxes need to be explicitly run with SCSI bus, 244 # otherwise boot process fails on mounting the disk 245 libvirt.disk_bus = "scsi" 246 elsif (distro.include?("arch")) 247 # Run generic/arch boxes explicitly with IDE bus, 248 # otherwise boot process fails on mounting the disk 249 libvirt.disk_bus = "ide" 250 else 251 libvirt.disk_bus = "virtio" 252 end 253 254 if ENV['SPDK_QEMU_EMULATOR'] 255 libvirt.emulator_path = ENV['SPDK_QEMU_EMULATOR'] 256 libvirt.machine_type = "pc" 257 end 258 259 # we put nvme_disk inside default pool to eliminate libvirt/SELinux Permissions Problems 260 # and to be able to run vagrant from user $HOME directory 261 262 # Loop to create all emulated disks set 263 emulated_nvme_types.each_with_index { |disk, index| 264 if disk == "nvme" 265 setup_nvme_disk(libvirt, disk, index) 266 elsif disk == "ocssd" 267 setup_ocssd_disk(libvirt, disk, index) 268 end 269 } 270 271 if ENV['VAGRANT_HUGE_MEM'] == "1" 272 libvirt.memorybacking :hugepages 273 end 274 275 # Optional field if we want use other storage pools than default 276 # libvirt.storage_pool_name = "vm" 277 end 278end 279 280################################################################################################# 281# Pick the right distro and bootstrap, default is fedora31 282distro = (ENV['SPDK_VAGRANT_DISTRO'] || "fedora31") 283provider = (ENV['SPDK_VAGRANT_PROVIDER'] || "virtualbox") 284 285# Get all variables for creating vm 286vmcpu = (ENV['SPDK_VAGRANT_VMCPU'] || 2) 287vmram = (ENV['SPDK_VAGRANT_VMRAM'] || 4096) 288nfs_sync_backend_distros = ['freebsd', 'clearlinux'] 289openstack_network = (ENV['SPDK_OPENSTACK_NETWORK'] || false) 290 291# generic/freebsd boxes do not work properly with vagrant-proxyconf and 292# have issues installing rsync and sshfs for syncing files. NFS is 293# pre-installed, so use it. 294# generic/fedora boxes on the other hand have problems running NFS 295# service so use sshfs+rsync combo instead. 296if (nfs_sync_backend_distros.any? { |d| distro.include?(d) }) 297 files_sync_backend = {type: :nfs, nfs_udp: false, mount_options: ['ro']} 298 plugins_sync_backend = {type: :nfs, nfs_udp: false} 299else 300 # Remove --copy-links from default rsync cmdline since we do want to sync 301 # actual symlinks as well. Also, since copy is made between host and its 302 # local VM we don't need to worry about saturating the local link so skip 303 # the compression to speed up the whole transfer. 304 files_sync_backend = {type: "rsync", rsync__auto: false, rsync__args: ["--archive", "--verbose", "--delete"]} 305 plugins_sync_backend = {type: :sshfs} 306end 307 308Vagrant.configure(2) do |config| 309 config.vm.box = get_box_type(distro) 310 config.vm.box_check_update = false 311 config.vm.synced_folder '.', '/vagrant', disabled: true 312 313 # Add network interface for openstack tests 314 if openstack_network == "1" 315 config.vm.network "private_network", ip: "10.0.2.15" 316 end 317 # Copy in the .gitconfig if it exists 318 copy_gitconfig(config) 319 320 # Copy the tsocks configuration file for use when installing some spdk test pool dependencies 321 copy_tsocks(config) 322 323 # Copy in the user's tools if they exists 324 copy_vagrant_tools(config,files_sync_backend) 325 326 # rsync the spdk directory if provision hasn't happened yet 327 # Warning: rsync does not work with freebsd boxes, so this step is disabled 328 copy_spdk_dir(config, files_sync_backend) 329 330 # rsync artifacts from build 331 copy_spdk_artifacts(config, plugins_sync_backend) 332 333 # vagrant-cachier caches apt/yum etc to speed subsequent 334 # vagrant up 335 setup_vagrant_cachier(config, plugins_sync_backend) 336 337 # use http proxy if avaiable 338 setup_proxy(config, distro) 339 340 # Clear CFLAGS in clear linux 341 clear_cflags(config) if distro == 'clearlinux' 342 343 # freebsd and clearlinux boxes in order to have spdk sources synced from 344 # host properly will use NFS with "ro" option enabled to prevent changes 345 # on host filesystem. 346 # To make sources usable in the guest VM we need to unmount them and use 347 # local copy. 348 make_spdk_local_copy_of_nfs(config,distro) if plugins_sync_backend[:type] == :nfs 349 350 # Setup SSH 351 setup_ssh(config) 352 353 # Virtualbox configuration 354 setup_virtualbox(config,vmcpu,vmram) 355 356 # This setup was Tested on Fedora 27 357 # libvirt configuration need modern Qemu(tested on 2.10) & vagrant-libvirt in version 0.0.39+ 358 # There are few limitation for SElinux - The file added outside libvirt must have proper SE ACL policy or setenforce 0 359 setup_libvirt(config,vmcpu,vmram,distro) 360 361 # provision the vm with all of the necessary spdk dependencies for running the autorun.sh tests 362 deploy_test_vm(config) 363end 364