Mercurial > vim
changeset 15036:73fef88aa0f9 v8.1.0529
patch 8.1.0529: flaky test sometimes fails in different ways
commit https://github.com/vim/vim/commit/f77af0e61339d553a0a41e3d19dd3fc89ba57fe8
Author: Bram Moolenaar <Bram@vim.org>
Date: Fri Nov 16 16:52:16 2018 +0100
patch 8.1.0529: flaky test sometimes fails in different ways
Problem: Flaky test sometimes fails in different ways.
Solution: When the second run gives a different error, try running the test
again, up to five times.
author | Bram Moolenaar <Bram@vim.org> |
---|---|
date | Fri, 16 Nov 2018 17:00:05 +0100 |
parents | bf0e699790b9 |
children | 4cbe76a4a1f6 |
files | src/testdir/runtest.vim src/version.c |
diffstat | 2 files changed, 37 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/src/testdir/runtest.vim +++ b/src/testdir/runtest.vim @@ -314,28 +314,47 @@ endif for s:test in sort(s:tests) " Silence, please! set belloff=all + let prev_error = '' + let total_errors = [] + let run_nr = 1 call RunTheTest(s:test) + " Repeat a flaky test. Give up when: + " - it fails again with the same message + " - it fails five times (with a different mesage) if len(v:errors) > 0 && index(s:flaky, s:test) >= 0 - call add(s:messages, 'Found errors in ' . s:test . ':') - call extend(s:messages, v:errors) - call add(s:messages, 'Flaky test failed, running it again') - let first_run = v:errors + while 1 + call add(s:messages, 'Found errors in ' . s:test . ':') + call extend(s:messages, v:errors) - " Flakiness is often caused by the system being very busy. Sleep a couple - " of seconds to have a higher chance of succeeding the second time. - sleep 2 + call add(total_errors, 'Run ' . run_nr . ':') + call extend(total_errors, v:errors) + + if run_nr == 5 || prev_error == v:errors[0] + call add(total_errors, 'Flaky test failed too often, giving up') + let v:errors = total_errors + break + endif - let v:errors = [] - call RunTheTest(s:test) - if len(v:errors) > 0 - let second_run = v:errors - let v:errors = ['First run:'] - call extend(v:errors, first_run) - call add(v:errors, 'Second run:') - call extend(v:errors, second_run) - endif + call add(s:messages, 'Flaky test failed, running it again') + + " Flakiness is often caused by the system being very busy. Sleep a + " couple of seconds to have a higher chance of succeeding the second + " time. + sleep 2 + + let prev_error = v:errors[0] + let v:errors = [] + let run_nr += 1 + + call RunTheTest(s:test) + + if len(v:errors) == 0 + " Test passed on rerun. + break + endif + endwhile endif call AfterTheTest()