Mercurial > vim
changeset 36408:fe62d8dd9102 draft v9.1.0820
patch 9.1.0820: tests: Mac OS tests are too flaky
Commit: https://github.com/vim/vim/commit/baab7c08653a4e1b7227d6426d96cab030ccf9e8
Author: Milly <milly.ca@gmail.com>
Date: Mon Oct 28 21:56:14 2024 +0100
patch 9.1.0820: tests: Mac OS tests are too flaky
Problem: tests: Mac OS tests are too flaky
Solution: Increase max test timeout to 25 minutes,
allow up to 10 retries on Mac OS runners,
refactor runtest.vim (Milly).
closes: #15940
Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Milly <milly.ca@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
author | Christian Brabandt <cb@256bit.org> |
---|---|
date | Mon, 28 Oct 2024 22:00:04 +0100 |
parents | 65921ec26072 |
children | a49b1d4a423d |
files | .github/workflows/ci.yml src/testdir/runtest.vim src/testdir/screendump.vim src/testdir/test_channel.vim src/version.c |
diffstat | 5 files changed, 63 insertions(+), 32 deletions(-) [+] |
line wrap: on
line diff
--- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -389,7 +389,7 @@ jobs: brew install diffutils - name: Test - timeout-minutes: 20 + timeout-minutes: 25 run: | make ${TEST}
--- a/src/testdir/runtest.vim +++ b/src/testdir/runtest.vim @@ -190,10 +190,6 @@ function GetAllocId(name) return lnum - top - 1 endfunc -if has('reltime') - let g:func_start = reltime() -endif - " Get the list of swap files in the current directory. func s:GetSwapFileList() let save_dir = &directory @@ -603,6 +599,16 @@ for g:testfunc in sort(s:tests) " A test can set g:test_is_flaky to retry running the test. let g:test_is_flaky = 0 + " A test can set g:max_run_nr to change the max retry count. + let g:max_run_nr = 5 + if has('mac') + let g:max_run_nr = 10 + endif + + " By default, give up if the same error occurs. A test can set + " g:giveup_same_error to 0 to not give up on the same error and keep trying. + let g:giveup_same_error = 1 + let starttime = strftime("%H:%M:%S") call RunTheTest(g:testfunc) @@ -618,10 +624,15 @@ for g:testfunc in sort(s:tests) call extend(s:messages, v:errors) let endtime = strftime("%H:%M:%S") - call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:') + if has('reltime') + let suffix = $' in{reltimestr(reltime(g:func_start))} seconds' + else + let suffix = '' + endif + call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}{suffix}:') call extend(total_errors, v:errors) - if g:run_nr >= 5 || prev_error == v:errors[0] + if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error == v:errors[0] call add(total_errors, 'Flaky test failed too often, giving up') let v:errors = total_errors break @@ -632,7 +643,8 @@ for g:testfunc in sort(s:tests) " Flakiness is often caused by the system being very busy. Sleep a " couple of seconds to have a higher chance of succeeding the second " time. - sleep 2 + let delay = g:run_nr * 2 + exe 'sleep' delay let prev_error = v:errors[0] let v:errors = []
--- a/src/testdir/screendump.vim +++ b/src/testdir/screendump.vim @@ -56,6 +56,7 @@ func VerifyScreenDump(buf, filename, opt " Starting a terminal to make a screendump is always considered flaky. let g:test_is_flaky = 1 + let g:giveup_same_error = 0 " wait for the pending updates to be handled. call TermWait(a:buf) @@ -83,41 +84,55 @@ func VerifyScreenDump(buf, filename, opt sleep 50m call delete(testfile) call term_dumpwrite(a:buf, testfile, a:options) + + if refdump->empty() + let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")' + call assert_report(msg) + " no point in retrying + let g:run_nr = 10 + return 1 + endif + let testdump = ReadAndFilter(testfile, filter) if refdump == testdump call delete(testfile) if did_mkdir call delete('failed', 'd') endif + if i > 0 + call remove(v:errors, -1) + endif break endif - if i == max_loops - " Leave the failed dump around for inspection. - if filereadable(reference) - let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")' - if a:0 == 1 - let msg = a:1 . ': ' . msg - endif - if len(testdump) != len(refdump) - let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump) - endif - else - let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")' - " no point in retrying - let g:run_nr = 10 + + " Leave the failed dump around for inspection. + let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")' + if a:0 == 1 + let msg = a:1 . ': ' . msg + endif + if len(testdump) != len(refdump) + let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump) + endif + for j in range(len(refdump)) + if j >= len(testdump) + break endif - for i in range(len(refdump)) - if i >= len(testdump) - break - endif - if testdump[i] != refdump[i] - let msg = msg . '; difference in line ' . (i + 1) . ': "' . testdump[i] . '"' - endif - endfor - call assert_report(msg) + if testdump[j] != refdump[j] + let msg = msg . '; difference in line ' . (j + 1) . ': "' . testdump[j] . '"' + endif + endfor + + " Always add the last error so that it is displayed on timeout. + " See TestTimeout() in runtest.vim. + if i > 0 + call remove(v:errors, -1) + endif + call assert_report(msg) + + let i += 1 + if i >= max_loops return 1 endif - let i += 1 endwhile return 0 endfunc
--- a/src/testdir/test_channel.vim +++ b/src/testdir/test_channel.vim @@ -2762,6 +2762,8 @@ func LspTests(port) endfunc func Test_channel_lsp_mode() + " The channel lsp mode test is flaky and gives the same error. + let g:giveup_same_error = 0 call RunServer('test_channel_lsp.py', 'LspTests', []) endfunc