changeset 15036:73fef88aa0f9 v8.1.0529

patch 8.1.0529: flaky test sometimes fails in different ways commit https://github.com/vim/vim/commit/f77af0e61339d553a0a41e3d19dd3fc89ba57fe8 Author: Bram Moolenaar <Bram@vim.org> Date: Fri Nov 16 16:52:16 2018 +0100 patch 8.1.0529: flaky test sometimes fails in different ways Problem: Flaky test sometimes fails in different ways. Solution: When the second run gives a different error, try running the test again, up to five times.
author Bram Moolenaar <Bram@vim.org>
date Fri, 16 Nov 2018 17:00:05 +0100
parents bf0e699790b9
children 4cbe76a4a1f6
files src/testdir/runtest.vim src/version.c
diffstat 2 files changed, 37 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -314,28 +314,47 @@ endif
 for s:test in sort(s:tests)
   " Silence, please!
   set belloff=all
+  let prev_error = ''
+  let total_errors = []
+  let run_nr = 1
 
   call RunTheTest(s:test)
 
+  " Repeat a flaky test.  Give up when:
+  " - it fails again with the same message
+  " - it fails five times (with a different mesage)
   if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
-    call add(s:messages, 'Found errors in ' . s:test . ':')
-    call extend(s:messages, v:errors)
-    call add(s:messages, 'Flaky test failed, running it again')
-    let first_run = v:errors
+    while 1
+      call add(s:messages, 'Found errors in ' . s:test . ':')
+      call extend(s:messages, v:errors)
 
-    " Flakiness is often caused by the system being very busy.  Sleep a couple
-    " of seconds to have a higher chance of succeeding the second time.
-    sleep 2
+      call add(total_errors, 'Run ' . run_nr . ':')
+      call extend(total_errors, v:errors)
+
+      if run_nr == 5 || prev_error == v:errors[0]
+        call add(total_errors, 'Flaky test failed too often, giving up')
+        let v:errors = total_errors
+        break
+      endif
 
-    let v:errors = []
-    call RunTheTest(s:test)
-    if len(v:errors) > 0
-      let second_run = v:errors
-      let v:errors = ['First run:']
-      call extend(v:errors, first_run)
-      call add(v:errors, 'Second run:')
-      call extend(v:errors, second_run)
-    endif
+      call add(s:messages, 'Flaky test failed, running it again')
+
+      " Flakiness is often caused by the system being very busy.  Sleep a
+      " couple of seconds to have a higher chance of succeeding the second
+      " time.
+      sleep 2
+
+      let prev_error = v:errors[0]
+      let v:errors = []
+      let run_nr += 1
+
+      call RunTheTest(s:test)
+
+      if len(v:errors) == 0
+        " Test passed on rerun.
+        break
+      endif
+    endwhile
   endif
 
   call AfterTheTest()
--- a/src/version.c
+++ b/src/version.c
@@ -793,6 +793,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    529,
+/**/
     528,
 /**/
     527,