Mercurial > vim
changeset 35992:c15f735c72e8 v9.1.0681
patch 9.1.0681: tests: Analyzing failed screendumps is hard
Commit: https://github.com/vim/vim/commit/6bff6a2fa449b9149eb2db4851e0b62a173b8748
Author: Aliaksei Budavei <0x000c70@gmail.com>
Date: Mon Aug 19 21:33:26 2024 +0200
patch 9.1.0681: tests: Analyzing failed screendumps is hard
Problem: tests: Analyzing failed screendumps is hard
Solution: Facilitate the viewing of rendered screendumps under src/
add some documentation on how to use the viewdumps.vim
script (Aliaksei Budavei)
With the submitted "viewdumps.vim" script, a few manual
steps in typical workflows (see below) can be automated.
The updated "README.txt" contains additional information.
============================================================
Reviewing LOCAL failed screendump tests can be arranged as
follows:
1) Run tests and generate screendumps:
------------------------------------------------------------
cd /path/to/fork/src/testdir
make
------------------------------------------------------------
2) Examine the screendumps from the "failed" directory:
------------------------------------------------------------
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
============================================================
Reviewing UPLOADED failed screendump tests can be arranged
as follows (it can be further locally scripted):
1) Fetch an artifact with failed screendumps from
"github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID".
2) Extract the archived files:
------------------------------------------------------------
unzip /tmp/failed-tests.zip -d /tmp
------------------------------------------------------------
3) Set up the "dumps" directory. Create a symlink to
"/path/to/fork/dirs/dumps" in the extracted directories so
that term_dumpdiff() can be used. (The lookup algorithm
resolves "dumps" for every loaded filename. So, with
"/tmp/src/testdir/failed/*.dump" files passed as script
arguments, the algorithm will make the files in
"/tmp/src/testdir/dumps" queried.)
------------------------------------------------------------
cd /path/to/fork
ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps
------------------------------------------------------------
4) Examine the extracted screendumps:
------------------------------------------------------------
./src/vim -u NONE -S src/testdir/viewdumps.vim \
/tmp/src/testdir/failed/*.dump
------------------------------------------------------------
5) Clean up:
------------------------------------------------------------
unlink /tmp/src/testdir/dumps
rm -rf /tmp/src
------------------------------------------------------------
============================================================
Reviewing SUBMITTED FOR PULL REQUEST screendump tests can be
arranged as follows (it can be further locally scripted):
1) List the fetched changeset and write the changed "dumps"
filenames to "/tmp/filelist":
------------------------------------------------------------
cd /path/to/fork
git switch prs/1234
git diff-index --relative=src/testdir/dumps/ \
--name-only prs/1234~1 > /tmp/filelist
------------------------------------------------------------
2) Reconcile relative filepaths, and copy next-to-be-updated
"dumps" files in the "failed" directory (note the missing
new screendumps, if any):
------------------------------------------------------------
git switch master
cd src/testdir/dumps
test -d ../failed || mkdir ../failed
cp -t ../failed $(cat /tmp/filelist)
------------------------------------------------------------
3) Remember about the introduced INVERTED relation between
"dumps" and "failed", i.e. the files to be committed are in
"dumps" already and their previous versions are in "failed";
therefore, copy the missing new screendumps from "dumps" to
"failed" (otherwise these won't be shown):
------------------------------------------------------------
git switch prs/1234
cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
------------------------------------------------------------
4) Examine the screendumps from the "failed" directory (new
screendumps will be shown with no difference between their
versions):
------------------------------------------------------------
cd ..
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
closes: #15515
Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
author | Christian Brabandt <cb@256bit.org> |
---|---|
date | Mon, 19 Aug 2024 21:45:05 +0200 |
parents | 96dd40a7c9a8 |
children | 1666cd879fd5 |
files | Filelist runtime/syntax/testdir/README.txt runtime/syntax/testdir/viewdumps.vim src/testdir/README.txt src/testdir/commondumps.vim src/testdir/viewdumps.vim src/version.c |
diffstat | 7 files changed, 231 insertions(+), 66 deletions(-) [+] |
line wrap: on
line diff
--- a/Filelist +++ b/Filelist @@ -185,6 +185,7 @@ SRC_ALL = \ src/testdir/sautest/autoload/*.vim \ src/testdir/testluaplugin/lua/testluaplugin/*.lua \ src/testdir/check.vim \ + src/testdir/commondumps.vim \ src/testdir/gui_init.vim \ src/testdir/gui_preinit.vim \ src/testdir/mouse.vim \ @@ -198,6 +199,7 @@ SRC_ALL = \ src/testdir/summarize.vim \ src/testdir/term_util.vim \ src/testdir/view_util.vim \ + src/testdir/viewdumps.vim \ src/testdir/test[0-9]*.ok \ src/testdir/test77a.ok \ src/testdir/test77a.com \
--- a/runtime/syntax/testdir/README.txt +++ b/runtime/syntax/testdir/README.txt @@ -124,8 +124,8 @@ verify that the tests fail. Then you kn test. -Viewing generated screendumps ------------------------------ +Viewing generated screendumps (local) +------------------------------------- You may also wish to look at the whole batch of failed screendumps after running "make test". Source the "viewdumps.vim" script for this task: @@ -151,5 +151,60 @@ At any time, you can add, list, and aban The listing of argument commands can be found under :help buffer-list. +Viewing generated screendumps (from a CI-uploaded artifact) +----------------------------------------------------------- + +After you have downloaded an artifact archive containing failed screendumps +and extracted its files in a temporary directory, you need to set up a "dumps" +directory by creating a symlink: + + cd /path/to/fork + ln -s $(pwd)/runtime/syntax/testdir/dumps \ + /tmp/runtime/syntax/testdir/dumps + +You can now examine the extracted screendumps: + + ./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \ + /tmp/runtime/syntax/testdir/failed/*.dump + + +Viewing generated screendumps (submitted for a pull request) +------------------------------------------------------------ + +First, you need to check out the topic branch with the proposed changes and +write down a difference list between the HEAD commit (index) and its parent +commit with respect to the changed "dumps" filenames: + + cd /path/to/fork + git switch prs/1234 + git diff-index --relative=runtime/syntax/testdir/dumps/ \ + --name-only prs/1234~1 > /tmp/filelist + +Then, you need to check out the master branch, change the current working +directory to reconcile relative filepaths written in the filenames list, copy +in the "failed" directory the old "dumps" files, whose names are on the same +list, and follow it by checking out the topic branch: + + git switch master + cd runtime/syntax/testdir/dumps + cp -t ../failed $(cat /tmp/filelist) + git switch prs/1234 + +Make note of any missing new screendumps. Please remember about the +introduced INVERTED relation between "dumps" and "failed", i.e. the files to +be committed are in "dumps" already and their old versions are in "failed". +Therefore, you need to copy the missing new screendumps from "dumps" to +"failed": + + cp -t ../failed foo_10.dump foo_11.dump foo_12.dump + +After you have changed the current working directory to its parent directory, +you can now examine the screendumps from the "failed" directory (note that new +screendumps will be shown with no difference between their versions): + + cd .. + ../../../src/vim --clean -S viewdumps.vim + + TODO: run test for one specific filetype TODO: test syncing by jumping around
--- a/runtime/syntax/testdir/viewdumps.vim +++ b/runtime/syntax/testdir/viewdumps.vim @@ -1,44 +1,9 @@ vim9script -# Support sourcing this script from this directory or any other directory in -# the direct path that leads to the project's root directory. -const failed_path: string = finddir('failed', getcwd() .. '/**', -1) - ->filter(((cwdpath: string) => (_: number, dirpath: string) => - cwdpath =~ '\<syntax\>' || dirpath =~ '\<syntax\>')(getcwd())) - ->get(-1, '') .. '/' -var error: string = null_string - -if failed_path == '/' - error = 'No such directory: "failed"' -else - const failed_fnames: string = failed_path .. readdir(failed_path, - (fname: string) => fname =~ '^.\+\.dump$') - ->join(' ' .. failed_path) - - if failed_fnames =~ 'failed/$' - error = 'No such file: "*.dump"' - else - exec ':0argedit ' .. failed_fnames - buffers - endif -endif - -def Render() - const failed_fname: string = bufname() - try - setlocal suffixesadd=.dump - const dumps_fname: string = findfile( - fnamemodify(failed_fname, ':p:t'), - fnamemodify(failed_fname, ':p:h:h') .. '/dumps') - if filereadable(dumps_fname) - term_dumpdiff(failed_fname, dumps_fname) - else - term_dumpload(failed_fname) - endif - finally - exec 'bwipeout ' .. failed_fname - endtry -enddef +exec 'source ' .. (((cwdpath: string) => cwdpath + ->strpart(0, cwdpath->strridx('/vim')))(getcwd())) + .. '/vim/src/testdir/commondumps.vim' +g:Init('\<syntax\>', -1) # THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE # LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED. @@ -56,22 +21,4 @@ set display=lastline ruler scrolloff=5 t # Anticipate non-Latin-1 characters in "input/" files. set encoding=utf-8 termencoding=utf-8 -autocmd_add([{ - replace: true, - group: 'viewdumps', - event: 'BufRead', - pattern: '*.dump', - cmd: 'Render()', -}]) - -# Unconditionally help, in case a list of filenames is passed to the command, -# the first terminal window with its BufRead event. -silent doautocmd viewdumps BufRead - -if error != null_string - # Instead of sleeping, fill half a window with blanks and prompt hit-enter. - echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error))) - error = null_string -endif - # vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
--- a/src/testdir/README.txt +++ b/src/testdir/README.txt @@ -97,8 +97,7 @@ tests are successful, then this file wil - To execute only specific test functions, add a second argument: - $ ../vim -u NONE -S runtest.vim test_channel.vim open_delay - + $ ../vim -u NONE -S runtest.vim test_channel.vim open_delay - To run all the tests: @@ -120,9 +119,82 @@ tests are successful, then this file wil $ make clean -# ANALYZE FAILED SCREENDUMPS FROM CI: + +VIEWING GENERATED SCREENDUMPS (local): + +You may also wish to look at the whole batch of failed screendumps after +running "make". Source the "viewdumps.vim" script for this task: + + $ ../vim -u NONE -S viewdumps.vim \ + [dumps/Test_xxd_*.dump ...] + +By default, all screendumps found in the "failed" directory will be added to +the argument list and then the first one will be loaded. Loaded screendumps +that bear filenames of screendumps found in the "dumps" directory will be +rendering the contents of any such pair of files and the difference between +them (:help term_dumpdiff()); otherwise, they will be rendering own contents +(:help term_dumpload()). Remember to execute :edit when occasionally you see +raw file contents instead of rendered. + +At any time, you can add, list, and abandon other screendumps: + + :$argedit dumps/Test_spell_*.dump + :args + :qall + +The listing of argument commands can be found under :help buffer-list. + + +VIEWING GENERATED SCREENDUMPS (from a CI-uploaded artifact): + +After you have downloaded an artifact archive containing failed screendumps +and extracted its files in a temporary directory, you need to set up a "dumps" +directory by creating a symlink: + + $ cd /path/to/fork + $ ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps + +You can now examine the extracted screendumps: + + $ ./src/vim -u NONE -S src/testdir/viewdumps.vim \ + /tmp/src/testdir/failed/*.dump -See the file ../../runtime/syntax/testdir/README.txt section -"Viewing generated screendumps" on how to analyze failed screen dumps -(from CI or locally) using the provided Vim script -../../runtime/syntax/testdir/viewdumps.vim in a more automatic way. + +VIEWING GENERATED SCREENDUMPS (submitted for a pull request): + +First, you need to check out the topic branch with the proposed changes and +write down a difference list between the HEAD commit (index) and its parent +commit with respect to the changed "dumps" filenames: + + $ cd /path/to/fork + $ git switch prs/1234 + $ git diff-index --relative=src/testdir/dumps/ \ + --name-only prs/1234~1 > /tmp/filelist + +Then, you need to check out the master branch, change the current working +directory to reconcile relative filepaths written in the filenames list, +possibly create an empty "failed" directory, copy in this directory the old +"dumps" files, whose names are on the same list, and follow it by checking out +the topic branch: + + $ git switch master + $ cd src/testdir/dumps + $ test -d ../failed || mkdir ../failed + $ cp -t ../failed $(cat /tmp/filelist) + $ git switch prs/1234 + +Make note of any missing new screendumps. Please remember about the +introduced INVERTED relation between "dumps" and "failed", i.e. the files to +be committed are in "dumps" already and their old versions are in "failed". +Therefore, you need to copy the missing new screendumps from "dumps" to +"failed": + + $ cp -t ../failed foo_10.dump foo_11.dump foo_12.dump + +After you have changed the current working directory to its parent directory, +you can now examine the screendumps from the "failed" directory (note that new +screendumps will be shown with no difference between their versions): + + $ cd .. + $ ../vim -u NONE -S viewdumps.vim +
new file mode 100644 --- /dev/null +++ b/src/testdir/commondumps.vim @@ -0,0 +1,76 @@ +vim9script + +# (Script-local.) +# +# Render a loaded screendump file or the difference of a loaded screendump +# file and its namesake file from the "dumps" directory. +def Render() + const failed_fname: string = bufname() + try + setlocal suffixesadd=.dump + const dumps_fname: string = findfile( + fnamemodify(failed_fname, ':p:t'), + fnamemodify(failed_fname, ':p:h:h') .. '/dumps') + if filereadable(dumps_fname) + term_dumpdiff(failed_fname, dumps_fname) + else + term_dumpload(failed_fname) + endif + finally + exec 'bwipeout ' .. failed_fname + endtry +enddef + +# Search for the "failed" directory in the passed _subtreedirname_ directories +# (usually "\<src\>" or "\<syntax\>") and, if found, select its passed _count_ +# occurence, add all its "*.dump" files to the argument list and list them; +# also define a BufRead autocommand that would invoke "Render()" for every +# "*.dump" file. +def g:Init(subtreedirname: string, count: number) + # Support sourcing this script from any directory in the direct path that + # leads to the project's root directory. + const failed_path: string = finddir('failed', getcwd() .. '/**', -1) + ->filter(((cwdpath: string, parentdirname: string) => + (_: number, dirpath: string) => + cwdpath =~ parentdirname || dirpath =~ parentdirname)( + getcwd(), + subtreedirname)) + ->get(count, '') .. '/' + var error: string = null_string + + if failed_path == '/' + error = 'No such directory: "failed"' + else + const failed_fnames: string = failed_path .. readdir(failed_path, + (fname: string) => fname =~ '^.\+\.dump$') + ->join(' ' .. failed_path) + + if failed_fnames =~ 'failed/$' + error = 'No such file: "*.dump"' + else + exec ':0argedit ' .. failed_fnames + buffers + endif + endif + + autocmd_add([{ + replace: true, + group: 'viewdumps', + event: 'BufRead', + pattern: '*.dump', + cmd: 'Render()', + }]) + + # Unconditionally help, in case a list of filenames is passed to the + # command, the first terminal window with its BufRead event. + silent doautocmd viewdumps BufRead + + if error != null_string + # Instead of sleeping, fill half a window with blanks and prompt + # hit-enter. + echom error .. repeat("\x20", + (winwidth(0) * (winheight(0) / 2) - strlen(error))) + endif +enddef + +# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
new file mode 100644 --- /dev/null +++ b/src/testdir/viewdumps.vim @@ -0,0 +1,11 @@ +vim9script + +exec 'source ' .. (((cwdpath: string) => cwdpath + ->strpart(0, cwdpath->strridx('/vim')))(getcwd())) + .. '/vim/src/testdir/commondumps.vim' +g:Init('\<src\>', 0) + +# Match ":language" of runtest.vim. +language messages C + +# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta: