Mercurial > vim
comparison src/testdir/runtest.vim @ 15040:da8a8cdcb8b0 v8.1.0531
patch 8.1.0531: flaky tests often fail with a common error message
commit https://github.com/vim/vim/commit/dbc0d2163aa5e090d5a0c83aea448803ddbab664
Author: Bram Moolenaar <Bram@vim.org>
Date: Fri Nov 16 18:22:45 2018 +0100
patch 8.1.0531: flaky tests often fail with a common error message
Problem: Flaky tests often fail with a common error message.
Solution: Add a pattern to match an error message indicating a flaky test.
| author | Bram Moolenaar <Bram@vim.org> |
|---|---|
| date | Fri, 16 Nov 2018 18:30:05 +0100 |
| parents | 26c266f6fbc6 |
| children | f47b5dfe5c6d |
comparison
equal
deleted
inserted
replaced
| 15039:8296f67e651c | 15040:da8a8cdcb8b0 |
|---|---|
| 271 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) | 271 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) |
| 272 endtry | 272 endtry |
| 273 endif | 273 endif |
| 274 | 274 |
| 275 " Names of flaky tests. | 275 " Names of flaky tests. |
| 276 let s:flaky = [ | 276 let s:flaky_tests = [ |
| 277 \ 'Test_call()', | 277 \ 'Test_call()', |
| 278 \ 'Test_channel_handler()', | 278 \ 'Test_channel_handler()', |
| 279 \ 'Test_client_server()', | 279 \ 'Test_client_server()', |
| 280 \ 'Test_close_and_exit_cb()', | 280 \ 'Test_close_and_exit_cb()', |
| 281 \ 'Test_close_callback()', | 281 \ 'Test_close_callback()', |
| 327 \ 'Test_with_partial_callback()', | 327 \ 'Test_with_partial_callback()', |
| 328 \ 'Test_zero_reply()', | 328 \ 'Test_zero_reply()', |
| 329 \ 'Test_zz1_terminal_in_gui()', | 329 \ 'Test_zz1_terminal_in_gui()', |
| 330 \ ] | 330 \ ] |
| 331 | 331 |
| 332 " Pattern indicating a common flaky test failure. | |
| 333 let s:flaky_errors_re = 'StopVimInTerminal' | |
| 334 | |
| 332 " Locate Test_ functions and execute them. | 335 " Locate Test_ functions and execute them. |
| 333 redir @q | 336 redir @q |
| 334 silent function /^Test_ | 337 silent function /^Test_ |
| 335 redir END | 338 redir END |
| 336 let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) | 339 let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) |
| 351 call RunTheTest(s:test) | 354 call RunTheTest(s:test) |
| 352 | 355 |
| 353 " Repeat a flaky test. Give up when: | 356 " Repeat a flaky test. Give up when: |
| 354 " - it fails again with the same message | 357 " - it fails again with the same message |
| 355 " - it fails five times (with a different mesage) | 358 " - it fails five times (with a different mesage) |
| 356 if len(v:errors) > 0 && index(s:flaky, s:test) >= 0 | 359 if len(v:errors) > 0 |
| 360 \ && (index(s:flaky_tests, s:test) >= 0 | |
| 361 \ || v:errors[0] =~ s:flaky_errors_re) | |
| 357 while 1 | 362 while 1 |
| 358 call add(s:messages, 'Found errors in ' . s:test . ':') | 363 call add(s:messages, 'Found errors in ' . s:test . ':') |
| 359 call extend(s:messages, v:errors) | 364 call extend(s:messages, v:errors) |
| 360 | 365 |
| 361 call add(total_errors, 'Run ' . run_nr . ':') | 366 call add(total_errors, 'Run ' . run_nr . ':') |