From bce8a257b8ab4142220fa5136d8f9577b65e9a73 Mon Sep 17 00:00:00 2001 From: Martin Pärtel Date: Tue, 14 Mar 2017 00:30:45 +0000 Subject: Retry `vagrant up` on error, to work around flakiness with parallelism. --- vagrant/test.rb | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'vagrant/test.rb') diff --git a/vagrant/test.rb b/vagrant/test.rb index eff2a82..b8489b6 100755 --- a/vagrant/test.rb +++ b/vagrant/test.rb @@ -38,6 +38,23 @@ unless specifically_selected_vms.empty? dirs = dirs.select { |dir| ARGV.include?(dir) } end +def with_retries(n, options = {}, &block) + options = { + sleep: 0, + sleep_jitter: 0, + }.merge options + loop do + begin + return block.call + rescue + raise $! if n <= 0 + puts "Retrying #{n} more times after catching: #{$!}" + end + sleep(options[:sleep] + (Random.rand - 0.5) * options[:sleep_jitter]) + n -= 1 + end +end + puts "Running #{dirs.size} VMs in parallel: #{dirs.join(' ')}" puts "You can follow the progress of each VM by tailing vagrant/*/test.log" puts "Note: if your terminal goes wonky after this command, type 'reset'" @@ -55,8 +72,12 @@ threads = dirs.map do |dir| Process.waitpid(pid) $?.success? end - unless run_and_log.call "vagrant up" - raise "vagrant up failed" + # parallel `vagrant up` commands can be flaky with VirtualBox due to lock contention, + # so give it a few retries. + with_retries(3, sleep: 1.0, sleep_jitter: 0.5) do + unless run_and_log.call "vagrant up" + raise "vagrant up failed" + end end unless run_and_log.call "vagrant rsync" raise "vagrant rsync failed" -- cgit v1.2.3