# HG changeset patch # User Christian Brabandt # Date 1723481104 -7200 # Node ID 043b537c852b591017814897085742c6fa4d7915 # Parent d3f93dc681af196610c9ecfd1d2e789f4c84e0b1 runtime(syntax-tests): Facilitate the viewing of rendered screendumps Commit: https://github.com/vim/vim/commit/d33afe12c6639d70fca82230df6b9fdee7365423 Author: Aliaksei Budavei <0x000c70@gmail.com> Date: Mon Aug 12 18:37:15 2024 +0200 runtime(syntax-tests): Facilitate the viewing of rendered screendumps With the submitted "viewdumps.vim" script, a few manual steps in typical workflows (see below) can be automated. The updated "README.txt" contains additional information. ============================================================ Reviewing LOCAL failed syntax tests can be arranged as follows: 1) Run tests and generate screendumps: ------------------------------------------------------------ cd /path/to/fork/runtime/syntax make clean test ------------------------------------------------------------ 2) Examine the screendumps from the "failed" directory: ------------------------------------------------------------ ../../src/vim --clean -S testdir/viewdumps.vim ------------------------------------------------------------ ============================================================ Reviewing UPLOADED failed syntax tests can be arranged as follows (it can be further locally scripted): 1) Fetch an artifact with failed screendumps from "github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID". 2) Extract the archived files: ------------------------------------------------------------ unzip /tmp/failed-tests.zip -d /tmp ------------------------------------------------------------ 3) Set up the "dumps" directory. Create a symlink to "/path/to/fork/dirs/dumps" in the extracted directories so that term_dumpdiff() can be used. (The lookup algorithm resolves "dumps" for every loaded filename. So, with "/tmp/runtime/syntax/testdir/failed/*.dump" files passed as script arguments, the algorithm will make the files in "/tmp/runtime/syntax/testdir/dumps" queried.) ------------------------------------------------------------ cd /path/to/fork ln -s $(pwd)/runtime/syntax/testdir/dumps \ /tmp/runtime/syntax/testdir/dumps ------------------------------------------------------------ 4) Examine the extracted screendumps: ------------------------------------------------------------ ./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \ /tmp/runtime/syntax/testdir/failed/*.dump ------------------------------------------------------------ 5) Clean up: ------------------------------------------------------------ unlink /tmp/runtime/syntax/testdir/dumps rm -rf /tmp/runtime ------------------------------------------------------------ ============================================================ Reviewing SUBMITTED FOR PULL REQUEST syntax tests can be arranged as follows (it can be further locally scripted): 1) List the fetched changeset and write the changed "dumps" filenames to "/tmp/filelist": ------------------------------------------------------------ cd /path/to/fork git switch prs/1234 git diff-index --relative=runtime/syntax/testdir/dumps/ \ --name-only prs/1234~1 > /tmp/filelist ------------------------------------------------------------ 2) Reconcile relative filepaths, and copy next-to-be-updated "dumps" files in the "failed" directory (note the missing new screendumps, if any): ------------------------------------------------------------ git switch master cd runtime/syntax/testdir/dumps cp -t ../failed $(cat /tmp/filelist) ------------------------------------------------------------ 3) Remember about the introduced INVERTED relation between "dumps" and "failed", i.e. the files to be committed are in "dumps" already and their previous versions are in "failed"; therefore, copy the missing new screendumps from "dumps" to "failed" (otherwise these won't be shown): ------------------------------------------------------------ git switch prs/1234 cp -t ../failed foo_10.dump foo_11.dump foo_12.dump ------------------------------------------------------------ 4) Examine the screendumps from the "failed" directory (new screendumps will be shown with no difference between their versions): ------------------------------------------------------------ cd .. ../../../src/vim --clean -S viewdumps.vim ------------------------------------------------------------ closes: #15476 Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com> Signed-off-by: Christian Brabandt diff --git a/.github/actions/screendump/action.yml b/.github/actions/screendump/action.yml --- a/.github/actions/screendump/action.yml +++ b/.github/actions/screendump/action.yml @@ -1,13 +1,13 @@ name: 'screendump' -description: "Upload failed syntax tests" +description: "Upload failed screendump tests" runs: using: "composite" steps: - - name: Upload failed syntax tests + - name: Upload failed tests uses: actions/upload-artifact@v4 with: # Name of the artifact to upload. - name: ${{ github.workflow }}-${{ github.job }}-${{ join(matrix.*, '-') }}-failed-syntax-tests + name: ${{ github.workflow }}-${{ github.job }}-${{ join(matrix.*, '-') }}-failed-tests # A file, directory or wildcard pattern that describes what # to upload. diff --git a/Filelist b/Filelist --- a/Filelist +++ b/Filelist @@ -836,6 +836,7 @@ RT_SCRIPTS = \ runtime/syntax/Makefile \ runtime/syntax/testdir/README.txt \ runtime/syntax/testdir/runtest.vim \ + runtime/syntax/testdir/viewdumps.vim \ runtime/syntax/testdir/ftplugin/*.* \ runtime/syntax/testdir/input/*.* \ runtime/syntax/testdir/input/selftestdir/* \ diff --git a/runtime/syntax/testdir/README.txt b/runtime/syntax/testdir/README.txt --- a/runtime/syntax/testdir/README.txt +++ b/runtime/syntax/testdir/README.txt @@ -124,6 +124,31 @@ verify that the tests fail. Then you kn test. +Viewing generated screendumps +----------------------------- + +You may also wish to look at the whole batch of failed screendumps after +running "make test". Source the "viewdumps.vim" script for this task: + + [VIMRUNTIME=../..] \ + ../../src/vim --clean -S testdir/viewdumps.vim \ + [testdir/dumps/java_*.dump ...] + +By default, all screendumps found in the "failed" directory will be added to +the argument list and then the first one will be loaded. Loaded screendumps +that bear filenames of screendumps found in the "dumps" directory will be +rendering the contents of any such pair of files and the difference between +them (:help term_dumpdiff()); otherwise, they will be rendering own contents +(:help term_dumpload()). Remember to execute :edit when occasionally you see +raw file contents instead of rendered. + +At any time, you can add, list, and abandon other screendumps: + + :$argedit testdir/dumps/java_*.dump + :args + :qall + +The listing of argument commands can be found under :help buffer-list. TODO: run test for one specific filetype diff --git a/runtime/syntax/testdir/runtest.vim b/runtime/syntax/testdir/runtest.vim --- a/runtime/syntax/testdir/runtest.vim +++ b/runtime/syntax/testdir/runtest.vim @@ -118,7 +118,7 @@ func RunTest() if exists("$VIM_SYNTAX_SELF_TESTING") let dirpath = 'input/selftestdir/' - let fnames = readdir(dirpath, {fname -> fname !~ '^README.txt$'}) + let fnames = readdir(dirpath, {fname -> fname !~ '^README\.txt$'}) else let dirpath = 'input/' let fnames = readdir(dirpath, {fname -> fname !~ '\~$' && fname =~ '^.\+\..\+$'}) @@ -409,6 +409,12 @@ func RunTest() call Message('OK: ' .. ok_count) call Message('FAILED: ' .. len(failed_tests) .. ': ' .. string(failed_tests)) call Message('skipped: ' .. skipped_count) + + if !empty(failed_tests) + call Message('') + call Message('View generated screendumps with "../../src/vim --clean -S testdir/viewdumps.vim"') + endif + call AppendMessages('== SUMMARY ==') if len(failed_tests) > 0 diff --git a/runtime/syntax/testdir/viewdumps.vim b/runtime/syntax/testdir/viewdumps.vim new file mode 100644 --- /dev/null +++ b/runtime/syntax/testdir/viewdumps.vim @@ -0,0 +1,77 @@ +vim9script + +# Support sourcing this script from this directory or any other directory in +# the direct path that leads to the project's root directory. +const failed_path: string = finddir('failed', getcwd() .. '/**', -1) + ->filter(((cwdpath: string) => (_: number, dirpath: string) => + cwdpath =~ '\' || dirpath =~ '\')(getcwd())) + ->get(-1, '') .. '/' +var error: string = null_string + +if failed_path == '/' + error = 'No such directory: "failed"' +else + const failed_fnames: string = failed_path .. readdir(failed_path, + (fname: string) => fname =~ '^.\+\.dump$') + ->join(' ' .. failed_path) + + if failed_fnames =~ 'failed/$' + error = 'No such file: "*.dump"' + else + exec ':0argedit ' .. failed_fnames + buffers + endif +endif + +def Render() + const failed_fname: string = bufname() + try + setlocal suffixesadd=.dump + const dumps_fname: string = findfile( + fnamemodify(failed_fname, ':p:t'), + fnamemodify(failed_fname, ':p:h:h') .. '/dumps') + if filereadable(dumps_fname) + term_dumpdiff(failed_fname, dumps_fname) + else + term_dumpload(failed_fname) + endif + finally + exec 'bwipeout ' .. failed_fname + endtry +enddef + +# THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE +# LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED. + +# Match "LC_ALL=C" of Makefile. +language C + +# Match the settings from term_util.vim#RunVimInTerminal(). +set t_Co=256 background=light +hi Normal ctermfg=NONE ctermbg=NONE + +# Match the settings from runtest.vim#Xtestscript#SetUpVim(). +set display=lastline ruler scrolloff=5 t_ZH= t_ZR= + +# Anticipate non-Latin-1 characters in "input/" files. +set encoding=utf-8 termencoding=utf-8 + +autocmd_add([{ + replace: true, + group: 'viewdumps', + event: 'BufRead', + pattern: '*.dump', + cmd: 'Render()', +}]) + +# Unconditionally help, in case a list of filenames is passed to the command, +# the first terminal window with its BufRead event. +silent doautocmd viewdumps BufRead + +if error != null_string + # Instead of sleeping, fill half a window with blanks and prompt hit-enter. + echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error))) + error = null_string +endif + +# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta: diff --git a/src/testdir/README.txt b/src/testdir/README.txt --- a/src/testdir/README.txt +++ b/src/testdir/README.txt @@ -119,3 +119,10 @@ tests are successful, then this file wil - To cleanup the temporary files after running the tests: $ make clean + +# ANALYZE FAILED SCREENDUMPS FROM CI: + +See the file ../../runtime/syntax/testdir/README.txt section +"Viewing generated screendumps" on how to analyze failed screen dumps +(from CI or locally) using the provided Vim script +../../runtime/syntax/testdir/viewdumps.vim in a more automatic way.