Mercurial > vim

comparison src/testdir/runtest.vim @ 15036:73fef88aa0f9 v8.1.0529

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
patch 8.1.0529: flaky test sometimes fails in different ways commit https://github.com/vim/vim/commit/f77af0e61339d553a0a41e3d19dd3fc89ba57fe8 Author: Bram Moolenaar <Bram@vim.org> Date: Fri Nov 16 16:52:16 2018 +0100 patch 8.1.0529: flaky test sometimes fails in different ways Problem: Flaky test sometimes fails in different ways. Solution: When the second run gives a different error, try running the test again, up to five times.
author Bram Moolenaar <Bram@vim.org>
date Fri, 16 Nov 2018 17:00:05 +0100
parents 563ca9655b8f
children 26c266f6fbc6
comparison
equal deleted inserted replaced
15035:bf0e699790b9 15036:73fef88aa0f9
312 312
313 " Execute the tests in alphabetical order. 313 " Execute the tests in alphabetical order.
314 for s:test in sort(s:tests) 314 for s:test in sort(s:tests)
315 " Silence, please! 315 " Silence, please!
316 set belloff=all 316 set belloff=all
317 let prev_error = ''
318 let total_errors = []
319 let run_nr = 1
317 320
318 call RunTheTest(s:test) 321 call RunTheTest(s:test)
319 322
323 " Repeat a flaky test. Give up when:
324 " - it fails again with the same message
325 " - it fails five times (with a different mesage)
320 if len(v:errors) > 0 && index(s:flaky, s:test) >= 0 326 if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
321 call add(s:messages, 'Found errors in ' . s:test . ':') 327 while 1
322 call extend(s:messages, v:errors) 328 call add(s:messages, 'Found errors in ' . s:test . ':')
323 call add(s:messages, 'Flaky test failed, running it again') 329 call extend(s:messages, v:errors)
324 let first_run = v:errors 330
325 331 call add(total_errors, 'Run ' . run_nr . ':')
326 " Flakiness is often caused by the system being very busy. Sleep a couple 332 call extend(total_errors, v:errors)
327 " of seconds to have a higher chance of succeeding the second time. 333
328 sleep 2 334 if run_nr == 5 || prev_error == v:errors[0]
329 335 call add(total_errors, 'Flaky test failed too often, giving up')
330 let v:errors = [] 336 let v:errors = total_errors
331 call RunTheTest(s:test) 337 break
332 if len(v:errors) > 0 338 endif
333 let second_run = v:errors 339
334 let v:errors = ['First run:'] 340 call add(s:messages, 'Flaky test failed, running it again')
335 call extend(v:errors, first_run) 341
336 call add(v:errors, 'Second run:') 342 " Flakiness is often caused by the system being very busy. Sleep a
337 call extend(v:errors, second_run) 343 " couple of seconds to have a higher chance of succeeding the second
338 endif 344 " time.
345 sleep 2
346
347 let prev_error = v:errors[0]
348 let v:errors = []
349 let run_nr += 1
350
351 call RunTheTest(s:test)
352
353 if len(v:errors) == 0
354 " Test passed on rerun.
355 break
356 endif
357 endwhile
339 endif 358 endif
340 359
341 call AfterTheTest() 360 call AfterTheTest()
342 endfor 361 endfor
343 362