Mercurial > vim
diff src/testdir/runtest.vim @ 32669:448aef880252
normalize line endings
author | Christian Brabandt <cb@256bit.org> |
---|---|
date | Mon, 26 Jun 2023 09:54:34 +0200 |
parents | 36519954bf67 |
children | 695b50472e85 |
line wrap: on
line diff
--- a/src/testdir/runtest.vim +++ b/src/testdir/runtest.vim @@ -1,622 +1,622 @@ -" This script is sourced while editing the .vim file with the tests. -" When the script is successful the .res file will be created. -" Errors are appended to the test.log file. -" -" To execute only specific test functions, add a second argument. It will be -" matched against the names of the Test_ function. E.g.: -" ../vim -u NONE -S runtest.vim test_channel.vim open_delay -" The output can be found in the "messages" file. -" -" If the environment variable $TEST_FILTER is set then only test functions -" matching this pattern are executed. E.g. for sh/bash: -" export TEST_FILTER=Test_channel -" For csh: -" setenv TEST_FILTER Test_channel -" -" If the environment variable $TEST_SKIP_PAT is set then test functions -" matching this pattern will be skipped. It's the opposite of $TEST_FILTER. -" -" While working on a test you can make $TEST_NO_RETRY non-empty to not retry: -" export TEST_NO_RETRY=yes -" -" To ignore failure for tests that are known to fail in a certain environment, -" set $TEST_MAY_FAIL to a comma separated list of function names. E.g. for -" sh/bash: -" export TEST_MAY_FAIL=Test_channel_one,Test_channel_other -" The failure report will then not be included in the test.log file and -" "make test" will not fail. -" -" The test script may contain anything, only functions that start with -" "Test_" are special. These will be invoked and should contain assert -" functions. See test_assert.vim for an example. -" -" It is possible to source other files that contain "Test_" functions. This -" can speed up testing, since Vim does not need to restart. But be careful -" that the tests do not interfere with each other. -" -" If an error cannot be detected properly with an assert function add the -" error to the v:errors list: -" call add(v:errors, 'test foo failed: Cannot find xyz') -" -" If preparation for each Test_ function is needed, define a SetUp function. -" It will be called before each Test_ function. -" -" If cleanup after each Test_ function is needed, define a TearDown function. -" It will be called after each Test_ function. -" -" When debugging a test it can be useful to add messages to v:errors: -" call add(v:errors, "this happened") - - -" Without the +eval feature we can't run these tests, bail out. -silent! while 0 - qa! -silent! endwhile - -" In the GUI we can always change the screen size. -if has('gui_running') - set columns=80 lines=25 -endif - -" Check that the screen size is at least 24 x 80 characters. -if &lines < 24 || &columns < 80 - let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters' - echoerr error - split test.log - $put =error - write - split messages - call append(line('$'), '') - call append(line('$'), 'From ' . expand('%') . ':') - call append(line('$'), error) - write - qa! -endif - -if has('reltime') - let s:run_start_time = reltime() - - if !filereadable('starttime') - " first test, store the overall test starting time - let s:test_start_time = localtime() - call writefile([string(s:test_start_time)], 'starttime') - else - " second or later test, read the overall test starting time - let s:test_start_time = readfile('starttime')[0]->str2nr() - endif -endif - -" Always use forward slashes. -set shellslash - -" Common with all tests on all systems. -source setup.vim - -" For consistency run all tests with 'nocompatible' set. -" This also enables use of line continuation. -set nocp viminfo+=nviminfo - -" Use utf-8 by default, instead of whatever the system default happens to be. -" Individual tests can overrule this at the top of the file and use -" g:orig_encoding if needed. -let g:orig_encoding = &encoding -set encoding=utf-8 - -" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for -" the test_name.vim file itself. Replace it here with a more restrictive one, -" so we still catch mistakes. -if has("win32") - " replace any '/' directory separators by '\\' - let s:test_script_fname = substitute(expand('%'), '/', '\\', 'g') -else - let s:test_script_fname = expand('%') -endif - -au! SwapExists * call HandleSwapExists() -func HandleSwapExists() - if exists('g:ignoreSwapExists') - if type(g:ignoreSwapExists) == v:t_string - let v:swapchoice = g:ignoreSwapExists - endif - return - endif - " Ignore finding a swap file for the test script (the user might be - " editing it and do ":make test_name") and the output file. - " Report finding another swap file and chose 'q' to avoid getting stuck. - if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname - let v:swapchoice = 'e' - else - call assert_report('Unexpected swap file: ' .. v:swapname) - let v:swapchoice = 'q' - endif -endfunc - -" Avoid stopping at the "hit enter" prompt -set nomore - -" Output all messages in English. -lang mess C - -" suppress menu translation -if has('gui_running') && exists('did_install_default_menus') - source $VIMRUNTIME/delmenu.vim - set langmenu=none - source $VIMRUNTIME/menu.vim -endif - -let s:srcdir = expand('%:p:h:h') - -if has('win32') - " avoid prompt that is long or contains a line break - let $PROMPT = '$P$G' - " On MS-Windows t_md and t_me are Vim specific escape sequences. - let s:t_bold = "\x1b[1m" - let s:t_normal = "\x1b[m" -else - let s:t_bold = &t_md - let s:t_normal = &t_me -endif - -if has('mac') - " In macOS, when starting a shell in a terminal, a bash deprecation warning - " message is displayed. This breaks the terminal test. Disable the warning - " message. - let $BASH_SILENCE_DEPRECATION_WARNING = 1 -endif - - -" Prepare for calling test_garbagecollect_now(). -let v:testing = 1 - -" By default, copy each buffer line into allocated memory, so that valgrind can -" detect accessing memory before and after it. -call test_override('alloc_lines', 1) - -" Support function: get the alloc ID by name. -function GetAllocId(name) - exe 'split ' . s:srcdir . '/alloc.h' - let top = search('typedef enum') - if top == 0 - call add(v:errors, 'typedef not found in alloc.h') - endif - let lnum = search('aid_' . a:name . ',') - if lnum == 0 - call add(v:errors, 'Alloc ID ' . a:name . ' not defined') - endif - close - return lnum - top - 1 -endfunc - -if has('reltime') - let g:func_start = reltime() -endif - -" Get the list of swap files in the current directory. -func s:GetSwapFileList() - let save_dir = &directory - let &directory = '.' - let files = swapfilelist() - let &directory = save_dir - - " remove a match with runtest.vim - let idx = indexof(files, 'v:val =~ "runtest.vim."') - if idx >= 0 - call remove(files, idx) - endif - - return files -endfunc - -" A previous (failed) test run may have left swap files behind. Delete them -" before running tests again, they might interfere. -for name in s:GetSwapFileList() - call delete(name) -endfor -unlet name - - -" Invoked when a test takes too much time. -func TestTimeout(id) - split test.log - call append(line('$'), '') - call append(line('$'), 'Test timed out: ' .. g:testfunc) - write - call add(v:errors, 'Test timed out: ' . g:testfunc) - - cquit! 42 -endfunc - -func RunTheTest(test) - let prefix = '' - if has('reltime') - let prefix = strftime('%M:%S', localtime() - s:test_start_time) .. ' ' - let g:func_start = reltime() - endif - echoconsole prefix .. 'Executing ' .. a:test - - if has('timers') - " No test should take longer than 30 seconds. If it takes longer we - " assume we are stuck and need to break out. - let test_timeout_timer = timer_start(30000, 'TestTimeout') - endif - - " Avoid stopping at the "hit enter" prompt - set nomore - - " Avoid a three second wait when a message is about to be overwritten by the - " mode message. - set noshowmode - - " Clear any overrides, except "alloc_lines". - call test_override('ALL', 0) - - " Some tests wipe out buffers. To be consistent, always wipe out all - " buffers. - %bwipe! - - " The test may change the current directory. Save and restore the - " directory after executing the test. - let save_cwd = getcwd() - - if exists("*SetUp") - try - call SetUp() - catch - call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) - endtry - endif - - au VimLeavePre * call EarlyExit(g:testfunc) - if a:test =~ 'Test_nocatch_' - " Function handles errors itself. This avoids skipping commands after the - " error. - let g:skipped_reason = '' - exe 'call ' . a:test - if g:skipped_reason != '' - call add(s:messages, ' Skipped') - call add(s:skipped, 'SKIPPED ' . a:test . ': ' . g:skipped_reason) - endif - else - try - exe 'call ' . a:test - catch /^\cskipped/ - call add(s:messages, ' Skipped') - call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) - catch - call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) - endtry - endif - au! VimLeavePre - - if a:test =~ '_terminal_' - " Terminal tests sometimes hang, give extra information - echoconsole 'After executing ' .. a:test - endif - - " In case 'insertmode' was set and something went wrong, make sure it is - " reset to avoid trouble with anything else. - set noinsertmode - - if exists("*TearDown") - try - call TearDown() - catch - call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) - endtry - endif - - if has('timers') - call timer_stop(test_timeout_timer) - endif - - " Clear any autocommands and put back the catch-all for SwapExists. - au! - au SwapExists * call HandleSwapExists() - - " Check for and close any stray popup windows. - if has('popupwin') - call assert_equal([], popup_list(), 'Popup is still present') - call popup_clear(1) - endif - - if filereadable('guidialogfile') - call add(v:errors, "Unexpected dialog: " .. readfile('guidialogfile')->join('<NL>')) - call delete('guidialogfile') - endif - - " Close any extra tab pages and windows and make the current one not modified. - while tabpagenr('$') > 1 - let winid = win_getid() - quit! - if winid == win_getid() - echoerr 'Could not quit window' - break - endif - endwhile - - while 1 - let wincount = winnr('$') - if wincount == 1 - break - endif - bwipe! - if wincount == winnr('$') - " Did not manage to close a window. - only! - break - endif - endwhile - - exe 'cd ' . save_cwd - - if a:test =~ '_terminal_' - " Terminal tests sometimes hang, give extra information - echoconsole 'Finished ' . a:test - endif - - let message = 'Executed ' . a:test - if has('reltime') - let message ..= repeat(' ', 50 - len(message)) - let time = reltime(g:func_start) - if reltimefloat(time) > 0.1 - let message = s:t_bold .. message - endif - let message ..= ' in ' .. reltimestr(time) .. ' seconds' - if reltimefloat(time) > 0.1 - let message ..= s:t_normal - endif - endif - call add(s:messages, message) - let s:done += 1 - - " close any split windows - while winnr('$') > 1 - bwipe! - endwhile - - " May be editing some buffer, wipe it out. Then we may end up in another - " buffer, continue until we end up in an empty no-name buffer without a swap - " file. - while bufname() != '' || execute('swapname') !~ 'No swap file' - let bn = bufnr() - - noswapfile bwipe! - - if bn == bufnr() - " avoid getting stuck in the same buffer - break - endif - endwhile - - " Check if the test has left any swap files behind. Delete them before - " running tests again, they might interfere. - let swapfiles = s:GetSwapFileList() - if len(swapfiles) > 0 - call add(s:messages, "Found swap files: " .. string(swapfiles)) - for name in swapfiles - call delete(name) - endfor - endif -endfunc - -func AfterTheTest(func_name) - if len(v:errors) > 0 - if match(s:may_fail_list, '^' .. a:func_name) >= 0 - let s:fail_expected += 1 - call add(s:errors_expected, 'Found errors in ' . g:testfunc . ':') - call extend(s:errors_expected, v:errors) - else - let s:fail += 1 - call add(s:errors, 'Found errors in ' . g:testfunc . ':') - call extend(s:errors, v:errors) - endif - let v:errors = [] - endif -endfunc - -func EarlyExit(test) - " It's OK for the test we use to test the quit detection. - if a:test != 'Test_zz_quit_detected()' - call add(v:errors, v:errmsg) - call add(v:errors, 'Test caused Vim to exit: ' . a:test) - endif - - call FinishTesting() -endfunc - -" This function can be called by a test if it wants to abort testing. -func FinishTesting() - call AfterTheTest('') - - " Don't write viminfo on exit. - set viminfo= - - " Clean up files created by setup.vim - call delete('XfakeHOME', 'rf') - - if s:fail == 0 && s:fail_expected == 0 - " Success, create the .res file so that make knows it's done. - exe 'split ' . fnamemodify(g:testname, ':r') . '.res' - write - endif - - if len(s:errors) > 0 - " Append errors to test.log - split test.log - call append(line('$'), '') - call append(line('$'), 'From ' . g:testname . ':') - call append(line('$'), s:errors) - write - endif - - if s:done == 0 - if s:filtered > 0 - if $TEST_FILTER != '' - let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'" - else - let message = "ALL tests match $TEST_SKIP_PAT: '" .. $TEST_SKIP_PAT .. "'" - endif - else - let message = 'NO tests executed' - endif - else - if s:filtered > 0 - call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER and $TEST_SKIP_PAT") - endif - let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') - endif - if s:done > 0 && has('reltime') - let message = s:t_bold .. message .. repeat(' ', 40 - len(message)) - let message ..= ' in ' .. reltimestr(reltime(s:run_start_time)) .. ' seconds' - let message ..= s:t_normal - endif - echo message - call add(s:messages, message) - if s:fail > 0 - let message = s:fail . ' FAILED:' - echo message - call add(s:messages, message) - call extend(s:messages, s:errors) - endif - if s:fail_expected > 0 - let message = s:fail_expected . ' FAILED (matching $TEST_MAY_FAIL):' - echo message - call add(s:messages, message) - call extend(s:messages, s:errors_expected) - endif - - " Add SKIPPED messages - call extend(s:messages, s:skipped) - - " Append messages to the file "messages" - split messages - call append(line('$'), '') - call append(line('$'), 'From ' . g:testname . ':') - call append(line('$'), s:messages) - write - - qall! -endfunc - -" Source the test script. First grab the file name, in case the script -" navigates away. g:testname can be used by the tests. -let g:testname = expand('%') -let s:done = 0 -let s:fail = 0 -let s:fail_expected = 0 -let s:errors = [] -let s:errors_expected = [] -let s:messages = [] -let s:skipped = [] -if expand('%') =~ 'test_vimscript.vim' - " this test has intentional errors, don't use try/catch. - source % -else - try - source % - catch /^\cskipped/ - call add(s:messages, ' Skipped') - call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) - catch - let s:fail += 1 - call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) - endtry -endif - -" Delete the .res file, it may change behavior for completion -call delete(fnamemodify(g:testname, ':r') .. '.res') - -" Locate Test_ functions and execute them. -redir @q -silent function /^Test_ -redir END -let s:tests = split(substitute(@q, '\(function\|def\) \(\k*()\)', '\2', 'g')) - -" If there is an extra argument filter the function names against it. -if argc() > 1 - let s:tests = filter(s:tests, 'v:val =~ argv(1)') -endif - -" If the environment variable $TEST_FILTER is set then filter the function -" names against it. -let s:filtered = 0 -if $TEST_FILTER != '' - let s:filtered = len(s:tests) - let s:tests = filter(s:tests, 'v:val =~ $TEST_FILTER') - let s:filtered -= len(s:tests) -endif - -let s:may_fail_list = [] -if $TEST_MAY_FAIL != '' - " Split the list at commas and add () to make it match g:testfunc. - let s:may_fail_list = split($TEST_MAY_FAIL, ',')->map({i, v -> v .. '()'}) -endif - -" Execute the tests in alphabetical order. -for g:testfunc in sort(s:tests) - if $TEST_SKIP_PAT != '' && g:testfunc =~ $TEST_SKIP_PAT - call add(s:messages, g:testfunc .. ' matches $TEST_SKIP_PAT') - let s:filtered += 1 - continue - endif - - " Silence, please! - set belloff=all - let prev_error = '' - let total_errors = [] - let g:run_nr = 1 - - " A test can set g:test_is_flaky to retry running the test. - let g:test_is_flaky = 0 - - let starttime = strftime("%H:%M:%S") - call RunTheTest(g:testfunc) - - " Repeat a flaky test. Give up when: - " - $TEST_NO_RETRY is not empty - " - it fails again with the same message - " - it fails five times (with a different message) - if len(v:errors) > 0 - \ && $TEST_NO_RETRY == '' - \ && g:test_is_flaky - while 1 - call add(s:messages, 'Found errors in ' .. g:testfunc .. ':') - call extend(s:messages, v:errors) - - let endtime = strftime("%H:%M:%S") - call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:') - call extend(total_errors, v:errors) - - if g:run_nr >= 5 || prev_error == v:errors[0] - call add(total_errors, 'Flaky test failed too often, giving up') - let v:errors = total_errors - break - endif - - call add(s:messages, 'Flaky test failed, running it again') - - " Flakiness is often caused by the system being very busy. Sleep a - " couple of seconds to have a higher chance of succeeding the second - " time. - sleep 2 - - let prev_error = v:errors[0] - let v:errors = [] - let g:run_nr += 1 - - let starttime = strftime("%H:%M:%S") - call RunTheTest(g:testfunc) - - if len(v:errors) == 0 - " Test passed on rerun. - break - endif - endwhile - endif - - call AfterTheTest(g:testfunc) -endfor - -call FinishTesting() - -" vim: shiftwidth=2 sts=2 expandtab +" This script is sourced while editing the .vim file with the tests. +" When the script is successful the .res file will be created. +" Errors are appended to the test.log file. +" +" To execute only specific test functions, add a second argument. It will be +" matched against the names of the Test_ function. E.g.: +" ../vim -u NONE -S runtest.vim test_channel.vim open_delay +" The output can be found in the "messages" file. +" +" If the environment variable $TEST_FILTER is set then only test functions +" matching this pattern are executed. E.g. for sh/bash: +" export TEST_FILTER=Test_channel +" For csh: +" setenv TEST_FILTER Test_channel +" +" If the environment variable $TEST_SKIP_PAT is set then test functions +" matching this pattern will be skipped. It's the opposite of $TEST_FILTER. +" +" While working on a test you can make $TEST_NO_RETRY non-empty to not retry: +" export TEST_NO_RETRY=yes +" +" To ignore failure for tests that are known to fail in a certain environment, +" set $TEST_MAY_FAIL to a comma separated list of function names. E.g. for +" sh/bash: +" export TEST_MAY_FAIL=Test_channel_one,Test_channel_other +" The failure report will then not be included in the test.log file and +" "make test" will not fail. +" +" The test script may contain anything, only functions that start with +" "Test_" are special. These will be invoked and should contain assert +" functions. See test_assert.vim for an example. +" +" It is possible to source other files that contain "Test_" functions. This +" can speed up testing, since Vim does not need to restart. But be careful +" that the tests do not interfere with each other. +" +" If an error cannot be detected properly with an assert function add the +" error to the v:errors list: +" call add(v:errors, 'test foo failed: Cannot find xyz') +" +" If preparation for each Test_ function is needed, define a SetUp function. +" It will be called before each Test_ function. +" +" If cleanup after each Test_ function is needed, define a TearDown function. +" It will be called after each Test_ function. +" +" When debugging a test it can be useful to add messages to v:errors: +" call add(v:errors, "this happened") + + +" Without the +eval feature we can't run these tests, bail out. +silent! while 0 + qa! +silent! endwhile + +" In the GUI we can always change the screen size. +if has('gui_running') + set columns=80 lines=25 +endif + +" Check that the screen size is at least 24 x 80 characters. +if &lines < 24 || &columns < 80 + let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters' + echoerr error + split test.log + $put =error + write + split messages + call append(line('$'), '') + call append(line('$'), 'From ' . expand('%') . ':') + call append(line('$'), error) + write + qa! +endif + +if has('reltime') + let s:run_start_time = reltime() + + if !filereadable('starttime') + " first test, store the overall test starting time + let s:test_start_time = localtime() + call writefile([string(s:test_start_time)], 'starttime') + else + " second or later test, read the overall test starting time + let s:test_start_time = readfile('starttime')[0]->str2nr() + endif +endif + +" Always use forward slashes. +set shellslash + +" Common with all tests on all systems. +source setup.vim + +" For consistency run all tests with 'nocompatible' set. +" This also enables use of line continuation. +set nocp viminfo+=nviminfo + +" Use utf-8 by default, instead of whatever the system default happens to be. +" Individual tests can overrule this at the top of the file and use +" g:orig_encoding if needed. +let g:orig_encoding = &encoding +set encoding=utf-8 + +" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for +" the test_name.vim file itself. Replace it here with a more restrictive one, +" so we still catch mistakes. +if has("win32") + " replace any '/' directory separators by '\\' + let s:test_script_fname = substitute(expand('%'), '/', '\\', 'g') +else + let s:test_script_fname = expand('%') +endif + +au! SwapExists * call HandleSwapExists() +func HandleSwapExists() + if exists('g:ignoreSwapExists') + if type(g:ignoreSwapExists) == v:t_string + let v:swapchoice = g:ignoreSwapExists + endif + return + endif + " Ignore finding a swap file for the test script (the user might be + " editing it and do ":make test_name") and the output file. + " Report finding another swap file and chose 'q' to avoid getting stuck. + if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname + let v:swapchoice = 'e' + else + call assert_report('Unexpected swap file: ' .. v:swapname) + let v:swapchoice = 'q' + endif +endfunc + +" Avoid stopping at the "hit enter" prompt +set nomore + +" Output all messages in English. +lang mess C + +" suppress menu translation +if has('gui_running') && exists('did_install_default_menus') + source $VIMRUNTIME/delmenu.vim + set langmenu=none + source $VIMRUNTIME/menu.vim +endif + +let s:srcdir = expand('%:p:h:h') + +if has('win32') + " avoid prompt that is long or contains a line break + let $PROMPT = '$P$G' + " On MS-Windows t_md and t_me are Vim specific escape sequences. + let s:t_bold = "\x1b[1m" + let s:t_normal = "\x1b[m" +else + let s:t_bold = &t_md + let s:t_normal = &t_me +endif + +if has('mac') + " In macOS, when starting a shell in a terminal, a bash deprecation warning + " message is displayed. This breaks the terminal test. Disable the warning + " message. + let $BASH_SILENCE_DEPRECATION_WARNING = 1 +endif + + +" Prepare for calling test_garbagecollect_now(). +let v:testing = 1 + +" By default, copy each buffer line into allocated memory, so that valgrind can +" detect accessing memory before and after it. +call test_override('alloc_lines', 1) + +" Support function: get the alloc ID by name. +function GetAllocId(name) + exe 'split ' . s:srcdir . '/alloc.h' + let top = search('typedef enum') + if top == 0 + call add(v:errors, 'typedef not found in alloc.h') + endif + let lnum = search('aid_' . a:name . ',') + if lnum == 0 + call add(v:errors, 'Alloc ID ' . a:name . ' not defined') + endif + close + return lnum - top - 1 +endfunc + +if has('reltime') + let g:func_start = reltime() +endif + +" Get the list of swap files in the current directory. +func s:GetSwapFileList() + let save_dir = &directory + let &directory = '.' + let files = swapfilelist() + let &directory = save_dir + + " remove a match with runtest.vim + let idx = indexof(files, 'v:val =~ "runtest.vim."') + if idx >= 0 + call remove(files, idx) + endif + + return files +endfunc + +" A previous (failed) test run may have left swap files behind. Delete them +" before running tests again, they might interfere. +for name in s:GetSwapFileList() + call delete(name) +endfor +unlet name + + +" Invoked when a test takes too much time. +func TestTimeout(id) + split test.log + call append(line('$'), '') + call append(line('$'), 'Test timed out: ' .. g:testfunc) + write + call add(v:errors, 'Test timed out: ' . g:testfunc) + + cquit! 42 +endfunc + +func RunTheTest(test) + let prefix = '' + if has('reltime') + let prefix = strftime('%M:%S', localtime() - s:test_start_time) .. ' ' + let g:func_start = reltime() + endif + echoconsole prefix .. 'Executing ' .. a:test + + if has('timers') + " No test should take longer than 30 seconds. If it takes longer we + " assume we are stuck and need to break out. + let test_timeout_timer = timer_start(30000, 'TestTimeout') + endif + + " Avoid stopping at the "hit enter" prompt + set nomore + + " Avoid a three second wait when a message is about to be overwritten by the + " mode message. + set noshowmode + + " Clear any overrides, except "alloc_lines". + call test_override('ALL', 0) + + " Some tests wipe out buffers. To be consistent, always wipe out all + " buffers. + %bwipe! + + " The test may change the current directory. Save and restore the + " directory after executing the test. + let save_cwd = getcwd() + + if exists("*SetUp") + try + call SetUp() + catch + call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) + endtry + endif + + au VimLeavePre * call EarlyExit(g:testfunc) + if a:test =~ 'Test_nocatch_' + " Function handles errors itself. This avoids skipping commands after the + " error. + let g:skipped_reason = '' + exe 'call ' . a:test + if g:skipped_reason != '' + call add(s:messages, ' Skipped') + call add(s:skipped, 'SKIPPED ' . a:test . ': ' . g:skipped_reason) + endif + else + try + exe 'call ' . a:test + catch /^\cskipped/ + call add(s:messages, ' Skipped') + call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) + catch + call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) + endtry + endif + au! VimLeavePre + + if a:test =~ '_terminal_' + " Terminal tests sometimes hang, give extra information + echoconsole 'After executing ' .. a:test + endif + + " In case 'insertmode' was set and something went wrong, make sure it is + " reset to avoid trouble with anything else. + set noinsertmode + + if exists("*TearDown") + try + call TearDown() + catch + call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) + endtry + endif + + if has('timers') + call timer_stop(test_timeout_timer) + endif + + " Clear any autocommands and put back the catch-all for SwapExists. + au! + au SwapExists * call HandleSwapExists() + + " Check for and close any stray popup windows. + if has('popupwin') + call assert_equal([], popup_list(), 'Popup is still present') + call popup_clear(1) + endif + + if filereadable('guidialogfile') + call add(v:errors, "Unexpected dialog: " .. readfile('guidialogfile')->join('<NL>')) + call delete('guidialogfile') + endif + + " Close any extra tab pages and windows and make the current one not modified. + while tabpagenr('$') > 1 + let winid = win_getid() + quit! + if winid == win_getid() + echoerr 'Could not quit window' + break + endif + endwhile + + while 1 + let wincount = winnr('$') + if wincount == 1 + break + endif + bwipe! + if wincount == winnr('$') + " Did not manage to close a window. + only! + break + endif + endwhile + + exe 'cd ' . save_cwd + + if a:test =~ '_terminal_' + " Terminal tests sometimes hang, give extra information + echoconsole 'Finished ' . a:test + endif + + let message = 'Executed ' . a:test + if has('reltime') + let message ..= repeat(' ', 50 - len(message)) + let time = reltime(g:func_start) + if reltimefloat(time) > 0.1 + let message = s:t_bold .. message + endif + let message ..= ' in ' .. reltimestr(time) .. ' seconds' + if reltimefloat(time) > 0.1 + let message ..= s:t_normal + endif + endif + call add(s:messages, message) + let s:done += 1 + + " close any split windows + while winnr('$') > 1 + bwipe! + endwhile + + " May be editing some buffer, wipe it out. Then we may end up in another + " buffer, continue until we end up in an empty no-name buffer without a swap + " file. + while bufname() != '' || execute('swapname') !~ 'No swap file' + let bn = bufnr() + + noswapfile bwipe! + + if bn == bufnr() + " avoid getting stuck in the same buffer + break + endif + endwhile + + " Check if the test has left any swap files behind. Delete them before + " running tests again, they might interfere. + let swapfiles = s:GetSwapFileList() + if len(swapfiles) > 0 + call add(s:messages, "Found swap files: " .. string(swapfiles)) + for name in swapfiles + call delete(name) + endfor + endif +endfunc + +func AfterTheTest(func_name) + if len(v:errors) > 0 + if match(s:may_fail_list, '^' .. a:func_name) >= 0 + let s:fail_expected += 1 + call add(s:errors_expected, 'Found errors in ' . g:testfunc . ':') + call extend(s:errors_expected, v:errors) + else + let s:fail += 1 + call add(s:errors, 'Found errors in ' . g:testfunc . ':') + call extend(s:errors, v:errors) + endif + let v:errors = [] + endif +endfunc + +func EarlyExit(test) + " It's OK for the test we use to test the quit detection. + if a:test != 'Test_zz_quit_detected()' + call add(v:errors, v:errmsg) + call add(v:errors, 'Test caused Vim to exit: ' . a:test) + endif + + call FinishTesting() +endfunc + +" This function can be called by a test if it wants to abort testing. +func FinishTesting() + call AfterTheTest('') + + " Don't write viminfo on exit. + set viminfo= + + " Clean up files created by setup.vim + call delete('XfakeHOME', 'rf') + + if s:fail == 0 && s:fail_expected == 0 + " Success, create the .res file so that make knows it's done. + exe 'split ' . fnamemodify(g:testname, ':r') . '.res' + write + endif + + if len(s:errors) > 0 + " Append errors to test.log + split test.log + call append(line('$'), '') + call append(line('$'), 'From ' . g:testname . ':') + call append(line('$'), s:errors) + write + endif + + if s:done == 0 + if s:filtered > 0 + if $TEST_FILTER != '' + let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'" + else + let message = "ALL tests match $TEST_SKIP_PAT: '" .. $TEST_SKIP_PAT .. "'" + endif + else + let message = 'NO tests executed' + endif + else + if s:filtered > 0 + call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER and $TEST_SKIP_PAT") + endif + let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') + endif + if s:done > 0 && has('reltime') + let message = s:t_bold .. message .. repeat(' ', 40 - len(message)) + let message ..= ' in ' .. reltimestr(reltime(s:run_start_time)) .. ' seconds' + let message ..= s:t_normal + endif + echo message + call add(s:messages, message) + if s:fail > 0 + let message = s:fail . ' FAILED:' + echo message + call add(s:messages, message) + call extend(s:messages, s:errors) + endif + if s:fail_expected > 0 + let message = s:fail_expected . ' FAILED (matching $TEST_MAY_FAIL):' + echo message + call add(s:messages, message) + call extend(s:messages, s:errors_expected) + endif + + " Add SKIPPED messages + call extend(s:messages, s:skipped) + + " Append messages to the file "messages" + split messages + call append(line('$'), '') + call append(line('$'), 'From ' . g:testname . ':') + call append(line('$'), s:messages) + write + + qall! +endfunc + +" Source the test script. First grab the file name, in case the script +" navigates away. g:testname can be used by the tests. +let g:testname = expand('%') +let s:done = 0 +let s:fail = 0 +let s:fail_expected = 0 +let s:errors = [] +let s:errors_expected = [] +let s:messages = [] +let s:skipped = [] +if expand('%') =~ 'test_vimscript.vim' + " this test has intentional errors, don't use try/catch. + source % +else + try + source % + catch /^\cskipped/ + call add(s:messages, ' Skipped') + call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) + catch + let s:fail += 1 + call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) + endtry +endif + +" Delete the .res file, it may change behavior for completion +call delete(fnamemodify(g:testname, ':r') .. '.res') + +" Locate Test_ functions and execute them. +redir @q +silent function /^Test_ +redir END +let s:tests = split(substitute(@q, '\(function\|def\) \(\k*()\)', '\2', 'g')) + +" If there is an extra argument filter the function names against it. +if argc() > 1 + let s:tests = filter(s:tests, 'v:val =~ argv(1)') +endif + +" If the environment variable $TEST_FILTER is set then filter the function +" names against it. +let s:filtered = 0 +if $TEST_FILTER != '' + let s:filtered = len(s:tests) + let s:tests = filter(s:tests, 'v:val =~ $TEST_FILTER') + let s:filtered -= len(s:tests) +endif + +let s:may_fail_list = [] +if $TEST_MAY_FAIL != '' + " Split the list at commas and add () to make it match g:testfunc. + let s:may_fail_list = split($TEST_MAY_FAIL, ',')->map({i, v -> v .. '()'}) +endif + +" Execute the tests in alphabetical order. +for g:testfunc in sort(s:tests) + if $TEST_SKIP_PAT != '' && g:testfunc =~ $TEST_SKIP_PAT + call add(s:messages, g:testfunc .. ' matches $TEST_SKIP_PAT') + let s:filtered += 1 + continue + endif + + " Silence, please! + set belloff=all + let prev_error = '' + let total_errors = [] + let g:run_nr = 1 + + " A test can set g:test_is_flaky to retry running the test. + let g:test_is_flaky = 0 + + let starttime = strftime("%H:%M:%S") + call RunTheTest(g:testfunc) + + " Repeat a flaky test. Give up when: + " - $TEST_NO_RETRY is not empty + " - it fails again with the same message + " - it fails five times (with a different message) + if len(v:errors) > 0 + \ && $TEST_NO_RETRY == '' + \ && g:test_is_flaky + while 1 + call add(s:messages, 'Found errors in ' .. g:testfunc .. ':') + call extend(s:messages, v:errors) + + let endtime = strftime("%H:%M:%S") + call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:') + call extend(total_errors, v:errors) + + if g:run_nr >= 5 || prev_error == v:errors[0] + call add(total_errors, 'Flaky test failed too often, giving up') + let v:errors = total_errors + break + endif + + call add(s:messages, 'Flaky test failed, running it again') + + " Flakiness is often caused by the system being very busy. Sleep a + " couple of seconds to have a higher chance of succeeding the second + " time. + sleep 2 + + let prev_error = v:errors[0] + let v:errors = [] + let g:run_nr += 1 + + let starttime = strftime("%H:%M:%S") + call RunTheTest(g:testfunc) + + if len(v:errors) == 0 + " Test passed on rerun. + break + endif + endwhile + endif + + call AfterTheTest(g:testfunc) +endfor + +call FinishTesting() + +" vim: shiftwidth=2 sts=2 expandtab