changeset 35949:043b537c852b

runtime(syntax-tests): Facilitate the viewing of rendered screendumps Commit: https://github.com/vim/vim/commit/d33afe12c6639d70fca82230df6b9fdee7365423 Author: Aliaksei Budavei <0x000c70@gmail.com> Date: Mon Aug 12 18:37:15 2024 +0200 runtime(syntax-tests): Facilitate the viewing of rendered screendumps With the submitted "viewdumps.vim" script, a few manual steps in typical workflows (see below) can be automated. The updated "README.txt" contains additional information. ============================================================ Reviewing LOCAL failed syntax tests can be arranged as follows: 1) Run tests and generate screendumps: ------------------------------------------------------------ cd /path/to/fork/runtime/syntax make clean test ------------------------------------------------------------ 2) Examine the screendumps from the "failed" directory: ------------------------------------------------------------ ../../src/vim --clean -S testdir/viewdumps.vim ------------------------------------------------------------ ============================================================ Reviewing UPLOADED failed syntax tests can be arranged as follows (it can be further locally scripted): 1) Fetch an artifact with failed screendumps from "github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID". 2) Extract the archived files: ------------------------------------------------------------ unzip /tmp/failed-tests.zip -d /tmp ------------------------------------------------------------ 3) Set up the "dumps" directory. Create a symlink to "/path/to/fork/dirs/dumps" in the extracted directories so that term_dumpdiff() can be used. (The lookup algorithm resolves "dumps" for every loaded filename. So, with "/tmp/runtime/syntax/testdir/failed/*.dump" files passed as script arguments, the algorithm will make the files in "/tmp/runtime/syntax/testdir/dumps" queried.) ------------------------------------------------------------ cd /path/to/fork ln -s $(pwd)/runtime/syntax/testdir/dumps \ /tmp/runtime/syntax/testdir/dumps ------------------------------------------------------------ 4) Examine the extracted screendumps: ------------------------------------------------------------ ./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \ /tmp/runtime/syntax/testdir/failed/*.dump ------------------------------------------------------------ 5) Clean up: ------------------------------------------------------------ unlink /tmp/runtime/syntax/testdir/dumps rm -rf /tmp/runtime ------------------------------------------------------------ ============================================================ Reviewing SUBMITTED FOR PULL REQUEST syntax tests can be arranged as follows (it can be further locally scripted): 1) List the fetched changeset and write the changed "dumps" filenames to "/tmp/filelist": ------------------------------------------------------------ cd /path/to/fork git switch prs/1234 git diff-index --relative=runtime/syntax/testdir/dumps/ \ --name-only prs/1234~1 > /tmp/filelist ------------------------------------------------------------ 2) Reconcile relative filepaths, and copy next-to-be-updated "dumps" files in the "failed" directory (note the missing new screendumps, if any): ------------------------------------------------------------ git switch master cd runtime/syntax/testdir/dumps cp -t ../failed $(cat /tmp/filelist) ------------------------------------------------------------ 3) Remember about the introduced INVERTED relation between "dumps" and "failed", i.e. the files to be committed are in "dumps" already and their previous versions are in "failed"; therefore, copy the missing new screendumps from "dumps" to "failed" (otherwise these won't be shown): ------------------------------------------------------------ git switch prs/1234 cp -t ../failed foo_10.dump foo_11.dump foo_12.dump ------------------------------------------------------------ 4) Examine the screendumps from the "failed" directory (new screendumps will be shown with no difference between their versions): ------------------------------------------------------------ cd .. ../../../src/vim --clean -S viewdumps.vim ------------------------------------------------------------ closes: #15476 Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com> Signed-off-by: Christian Brabandt <cb@256bit.org>
author Christian Brabandt <cb@256bit.org>
date Mon, 12 Aug 2024 18:45:04 +0200
parents d3f93dc681af
children dc59abfe9976
files .github/actions/screendump/action.yml Filelist runtime/syntax/testdir/README.txt runtime/syntax/testdir/runtest.vim runtime/syntax/testdir/viewdumps.vim src/testdir/README.txt
diffstat 6 files changed, 120 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/.github/actions/screendump/action.yml
+++ b/.github/actions/screendump/action.yml
@@ -1,13 +1,13 @@
 name: 'screendump'
-description: "Upload failed syntax tests"
+description: "Upload failed screendump tests"
 runs:
   using: "composite"
   steps:
-    - name: Upload failed syntax tests
+    - name: Upload failed tests
       uses: actions/upload-artifact@v4
       with:
         # Name of the artifact to upload.
-        name: ${{ github.workflow }}-${{ github.job }}-${{ join(matrix.*, '-') }}-failed-syntax-tests
+        name: ${{ github.workflow }}-${{ github.job }}-${{ join(matrix.*, '-') }}-failed-tests
 
         # A file, directory or wildcard pattern that describes what
         # to upload.
--- a/Filelist
+++ b/Filelist
@@ -836,6 +836,7 @@ RT_SCRIPTS =	\
 		runtime/syntax/Makefile \
 		runtime/syntax/testdir/README.txt \
 		runtime/syntax/testdir/runtest.vim \
+		runtime/syntax/testdir/viewdumps.vim \
 		runtime/syntax/testdir/ftplugin/*.* \
 		runtime/syntax/testdir/input/*.* \
 		runtime/syntax/testdir/input/selftestdir/* \
--- a/runtime/syntax/testdir/README.txt
+++ b/runtime/syntax/testdir/README.txt
@@ -124,6 +124,31 @@ verify that the tests fail.  Then you kn
 test.
 
 
+Viewing generated screendumps
+-----------------------------
+
+You may also wish to look at the whole batch of failed screendumps after
+running "make test".  Source the "viewdumps.vim" script for this task:
+
+	[VIMRUNTIME=../..] \
+	../../src/vim --clean -S testdir/viewdumps.vim \
+				[testdir/dumps/java_*.dump ...]
+
+By default, all screendumps found in the "failed" directory will be added to
+the argument list and then the first one will be loaded.  Loaded screendumps
+that bear filenames of screendumps found in the "dumps" directory will be
+rendering the contents of any such pair of files and the difference between
+them (:help term_dumpdiff()); otherwise, they will be rendering own contents
+(:help term_dumpload()).  Remember to execute :edit when occasionally you see
+raw file contents instead of rendered.
+
+At any time, you can add, list, and abandon other screendumps:
+
+	:$argedit testdir/dumps/java_*.dump
+	:args
+	:qall
+
+The listing of argument commands can be found under :help buffer-list.
 
 
 TODO: run test for one specific filetype
--- a/runtime/syntax/testdir/runtest.vim
+++ b/runtime/syntax/testdir/runtest.vim
@@ -118,7 +118,7 @@ func RunTest()
 
   if exists("$VIM_SYNTAX_SELF_TESTING")
     let dirpath = 'input/selftestdir/'
-    let fnames = readdir(dirpath, {fname -> fname !~ '^README.txt$'})
+    let fnames = readdir(dirpath, {fname -> fname !~ '^README\.txt$'})
   else
     let dirpath = 'input/'
     let fnames = readdir(dirpath, {fname -> fname !~ '\~$' && fname =~ '^.\+\..\+$'})
@@ -409,6 +409,12 @@ func RunTest()
   call Message('OK: ' .. ok_count)
   call Message('FAILED: ' .. len(failed_tests) .. ': ' .. string(failed_tests))
   call Message('skipped: ' .. skipped_count)
+
+  if !empty(failed_tests)
+    call Message('')
+    call Message('View generated screendumps with "../../src/vim --clean -S testdir/viewdumps.vim"')
+  endif
+
   call AppendMessages('== SUMMARY ==')
 
   if len(failed_tests) > 0
new file mode 100644
--- /dev/null
+++ b/runtime/syntax/testdir/viewdumps.vim
@@ -0,0 +1,77 @@
+vim9script
+
+# Support sourcing this script from this directory or any other directory in
+# the direct path that leads to the project's root directory.
+const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
+  ->filter(((cwdpath: string) => (_: number, dirpath: string) =>
+	cwdpath =~ '\<syntax\>' || dirpath =~ '\<syntax\>')(getcwd()))
+  ->get(-1, '') .. '/'
+var error: string = null_string
+
+if failed_path == '/'
+  error = 'No such directory: "failed"'
+else
+  const failed_fnames: string = failed_path .. readdir(failed_path,
+	  (fname: string) => fname =~ '^.\+\.dump$')
+    ->join(' ' .. failed_path)
+
+  if failed_fnames =~ 'failed/$'
+    error = 'No such file: "*.dump"'
+  else
+    exec ':0argedit ' .. failed_fnames
+    buffers
+  endif
+endif
+
+def Render()
+  const failed_fname: string = bufname()
+  try
+    setlocal suffixesadd=.dump
+    const dumps_fname: string = findfile(
+			fnamemodify(failed_fname, ':p:t'),
+			fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
+    if filereadable(dumps_fname)
+      term_dumpdiff(failed_fname, dumps_fname)
+    else
+      term_dumpload(failed_fname)
+    endif
+  finally
+    exec 'bwipeout ' .. failed_fname
+  endtry
+enddef
+
+# THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE
+# LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED.
+
+# Match "LC_ALL=C" of Makefile.
+language C
+
+# Match the settings from term_util.vim#RunVimInTerminal().
+set t_Co=256 background=light
+hi Normal ctermfg=NONE ctermbg=NONE
+
+# Match the settings from runtest.vim#Xtestscript#SetUpVim().
+set display=lastline ruler scrolloff=5 t_ZH= t_ZR=
+
+# Anticipate non-Latin-1 characters in "input/" files.
+set encoding=utf-8 termencoding=utf-8
+
+autocmd_add([{
+  replace:	true,
+  group:	'viewdumps',
+  event:	'BufRead',
+  pattern:	'*.dump',
+  cmd:		'Render()',
+}])
+
+# Unconditionally help, in case a list of filenames is passed to the command,
+# the first terminal window with its BufRead event.
+silent doautocmd viewdumps BufRead
+
+if error != null_string
+  # Instead of sleeping, fill half a window with blanks and prompt hit-enter.
+  echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error)))
+  error = null_string
+endif
+
+# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
--- a/src/testdir/README.txt
+++ b/src/testdir/README.txt
@@ -119,3 +119,10 @@ tests are successful, then this file wil
 - To cleanup the temporary files after running the tests:
 
     $ make clean
+
+# ANALYZE FAILED SCREENDUMPS FROM CI:
+
+See the file ../../runtime/syntax/testdir/README.txt section
+"Viewing generated screendumps" on how to analyze failed screen dumps
+(from CI or locally) using the provided Vim script
+../../runtime/syntax/testdir/viewdumps.vim in a more automatic way.