changeset 35992:c15f735c72e8 v9.1.0681

patch 9.1.0681: tests: Analyzing failed screendumps is hard Commit: https://github.com/vim/vim/commit/6bff6a2fa449b9149eb2db4851e0b62a173b8748 Author: Aliaksei Budavei <0x000c70@gmail.com> Date: Mon Aug 19 21:33:26 2024 +0200 patch 9.1.0681: tests: Analyzing failed screendumps is hard Problem: tests: Analyzing failed screendumps is hard Solution: Facilitate the viewing of rendered screendumps under src/ add some documentation on how to use the viewdumps.vim script (Aliaksei Budavei) With the submitted "viewdumps.vim" script, a few manual steps in typical workflows (see below) can be automated. The updated "README.txt" contains additional information. ============================================================ Reviewing LOCAL failed screendump tests can be arranged as follows: 1) Run tests and generate screendumps: ------------------------------------------------------------ cd /path/to/fork/src/testdir make ------------------------------------------------------------ 2) Examine the screendumps from the "failed" directory: ------------------------------------------------------------ ../vim -u NONE -S viewdumps.vim ------------------------------------------------------------ ============================================================ Reviewing UPLOADED failed screendump tests can be arranged as follows (it can be further locally scripted): 1) Fetch an artifact with failed screendumps from "github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID". 2) Extract the archived files: ------------------------------------------------------------ unzip /tmp/failed-tests.zip -d /tmp ------------------------------------------------------------ 3) Set up the "dumps" directory. Create a symlink to "/path/to/fork/dirs/dumps" in the extracted directories so that term_dumpdiff() can be used. (The lookup algorithm resolves "dumps" for every loaded filename. So, with "/tmp/src/testdir/failed/*.dump" files passed as script arguments, the algorithm will make the files in "/tmp/src/testdir/dumps" queried.) ------------------------------------------------------------ cd /path/to/fork ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps ------------------------------------------------------------ 4) Examine the extracted screendumps: ------------------------------------------------------------ ./src/vim -u NONE -S src/testdir/viewdumps.vim \ /tmp/src/testdir/failed/*.dump ------------------------------------------------------------ 5) Clean up: ------------------------------------------------------------ unlink /tmp/src/testdir/dumps rm -rf /tmp/src ------------------------------------------------------------ ============================================================ Reviewing SUBMITTED FOR PULL REQUEST screendump tests can be arranged as follows (it can be further locally scripted): 1) List the fetched changeset and write the changed "dumps" filenames to "/tmp/filelist": ------------------------------------------------------------ cd /path/to/fork git switch prs/1234 git diff-index --relative=src/testdir/dumps/ \ --name-only prs/1234~1 > /tmp/filelist ------------------------------------------------------------ 2) Reconcile relative filepaths, and copy next-to-be-updated "dumps" files in the "failed" directory (note the missing new screendumps, if any): ------------------------------------------------------------ git switch master cd src/testdir/dumps test -d ../failed || mkdir ../failed cp -t ../failed $(cat /tmp/filelist) ------------------------------------------------------------ 3) Remember about the introduced INVERTED relation between "dumps" and "failed", i.e. the files to be committed are in "dumps" already and their previous versions are in "failed"; therefore, copy the missing new screendumps from "dumps" to "failed" (otherwise these won't be shown): ------------------------------------------------------------ git switch prs/1234 cp -t ../failed foo_10.dump foo_11.dump foo_12.dump ------------------------------------------------------------ 4) Examine the screendumps from the "failed" directory (new screendumps will be shown with no difference between their versions): ------------------------------------------------------------ cd .. ../vim -u NONE -S viewdumps.vim ------------------------------------------------------------ closes: #15515 Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com> Signed-off-by: Christian Brabandt <cb@256bit.org>
author Christian Brabandt <cb@256bit.org>
date Mon, 19 Aug 2024 21:45:05 +0200
parents 96dd40a7c9a8
children 1666cd879fd5
files Filelist runtime/syntax/testdir/README.txt runtime/syntax/testdir/viewdumps.vim src/testdir/README.txt src/testdir/commondumps.vim src/testdir/viewdumps.vim src/version.c
diffstat 7 files changed, 231 insertions(+), 66 deletions(-) [+]
line wrap: on
line diff
--- a/Filelist
+++ b/Filelist
@@ -185,6 +185,7 @@ SRC_ALL =	\
 		src/testdir/sautest/autoload/*.vim \
 		src/testdir/testluaplugin/lua/testluaplugin/*.lua \
 		src/testdir/check.vim \
+		src/testdir/commondumps.vim \
 		src/testdir/gui_init.vim \
 		src/testdir/gui_preinit.vim \
 		src/testdir/mouse.vim \
@@ -198,6 +199,7 @@ SRC_ALL =	\
 		src/testdir/summarize.vim \
 		src/testdir/term_util.vim \
 		src/testdir/view_util.vim \
+		src/testdir/viewdumps.vim \
 		src/testdir/test[0-9]*.ok \
 		src/testdir/test77a.ok \
 		src/testdir/test77a.com \
--- a/runtime/syntax/testdir/README.txt
+++ b/runtime/syntax/testdir/README.txt
@@ -124,8 +124,8 @@ verify that the tests fail.  Then you kn
 test.
 
 
-Viewing generated screendumps
------------------------------
+Viewing generated screendumps (local)
+-------------------------------------
 
 You may also wish to look at the whole batch of failed screendumps after
 running "make test".  Source the "viewdumps.vim" script for this task:
@@ -151,5 +151,60 @@ At any time, you can add, list, and aban
 The listing of argument commands can be found under :help buffer-list.
 
 
+Viewing generated screendumps (from a CI-uploaded artifact)
+-----------------------------------------------------------
+
+After you have downloaded an artifact archive containing failed screendumps
+and extracted its files in a temporary directory, you need to set up a "dumps"
+directory by creating a symlink:
+
+	cd /path/to/fork
+	ln -s $(pwd)/runtime/syntax/testdir/dumps \
+				/tmp/runtime/syntax/testdir/dumps
+
+You can now examine the extracted screendumps:
+
+	./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \
+				/tmp/runtime/syntax/testdir/failed/*.dump
+
+
+Viewing generated screendumps (submitted for a pull request)
+------------------------------------------------------------
+
+First, you need to check out the topic branch with the proposed changes and
+write down a difference list between the HEAD commit (index) and its parent
+commit with respect to the changed "dumps" filenames:
+
+	cd /path/to/fork
+	git switch prs/1234
+	git diff-index --relative=runtime/syntax/testdir/dumps/ \
+				--name-only prs/1234~1 > /tmp/filelist
+
+Then, you need to check out the master branch, change the current working
+directory to reconcile relative filepaths written in the filenames list, copy
+in the "failed" directory the old "dumps" files, whose names are on the same
+list, and follow it by checking out the topic branch:
+
+	git switch master
+	cd runtime/syntax/testdir/dumps
+	cp -t ../failed $(cat /tmp/filelist)
+	git switch prs/1234
+
+Make note of any missing new screendumps.  Please remember about the
+introduced INVERTED relation between "dumps" and "failed", i.e. the files to
+be committed are in "dumps" already and their old versions are in "failed".
+Therefore, you need to copy the missing new screendumps from "dumps" to
+"failed":
+
+	cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
+
+After you have changed the current working directory to its parent directory,
+you can now examine the screendumps from the "failed" directory (note that new
+screendumps will be shown with no difference between their versions):
+
+	cd ..
+	../../../src/vim --clean -S viewdumps.vim
+
+
 TODO: run test for one specific filetype
 TODO: test syncing by jumping around
--- a/runtime/syntax/testdir/viewdumps.vim
+++ b/runtime/syntax/testdir/viewdumps.vim
@@ -1,44 +1,9 @@
 vim9script
 
-# Support sourcing this script from this directory or any other directory in
-# the direct path that leads to the project's root directory.
-const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
-  ->filter(((cwdpath: string) => (_: number, dirpath: string) =>
-	cwdpath =~ '\<syntax\>' || dirpath =~ '\<syntax\>')(getcwd()))
-  ->get(-1, '') .. '/'
-var error: string = null_string
-
-if failed_path == '/'
-  error = 'No such directory: "failed"'
-else
-  const failed_fnames: string = failed_path .. readdir(failed_path,
-	  (fname: string) => fname =~ '^.\+\.dump$')
-    ->join(' ' .. failed_path)
-
-  if failed_fnames =~ 'failed/$'
-    error = 'No such file: "*.dump"'
-  else
-    exec ':0argedit ' .. failed_fnames
-    buffers
-  endif
-endif
-
-def Render()
-  const failed_fname: string = bufname()
-  try
-    setlocal suffixesadd=.dump
-    const dumps_fname: string = findfile(
-			fnamemodify(failed_fname, ':p:t'),
-			fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
-    if filereadable(dumps_fname)
-      term_dumpdiff(failed_fname, dumps_fname)
-    else
-      term_dumpload(failed_fname)
-    endif
-  finally
-    exec 'bwipeout ' .. failed_fname
-  endtry
-enddef
+exec 'source ' .. (((cwdpath: string) => cwdpath
+  ->strpart(0, cwdpath->strridx('/vim')))(getcwd()))
+  .. '/vim/src/testdir/commondumps.vim'
+g:Init('\<syntax\>', -1)
 
 # THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE
 # LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED.
@@ -56,22 +21,4 @@ set display=lastline ruler scrolloff=5 t
 # Anticipate non-Latin-1 characters in "input/" files.
 set encoding=utf-8 termencoding=utf-8
 
-autocmd_add([{
-  replace:	true,
-  group:	'viewdumps',
-  event:	'BufRead',
-  pattern:	'*.dump',
-  cmd:		'Render()',
-}])
-
-# Unconditionally help, in case a list of filenames is passed to the command,
-# the first terminal window with its BufRead event.
-silent doautocmd viewdumps BufRead
-
-if error != null_string
-  # Instead of sleeping, fill half a window with blanks and prompt hit-enter.
-  echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error)))
-  error = null_string
-endif
-
 # vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
--- a/src/testdir/README.txt
+++ b/src/testdir/README.txt
@@ -97,8 +97,7 @@ tests are successful, then this file wil
 
 - To execute only specific test functions, add a second argument:
 
-	 $ ../vim -u NONE -S runtest.vim test_channel.vim open_delay
-
+    $ ../vim -u NONE -S runtest.vim test_channel.vim open_delay
 
 - To run all the tests:
 
@@ -120,9 +119,82 @@ tests are successful, then this file wil
 
     $ make clean
 
-# ANALYZE FAILED SCREENDUMPS FROM CI:
+
+VIEWING GENERATED SCREENDUMPS (local):
+
+You may also wish to look at the whole batch of failed screendumps after
+running "make".  Source the "viewdumps.vim" script for this task:
+
+    $ ../vim -u NONE -S viewdumps.vim \
+				[dumps/Test_xxd_*.dump ...]
+
+By default, all screendumps found in the "failed" directory will be added to
+the argument list and then the first one will be loaded.  Loaded screendumps
+that bear filenames of screendumps found in the "dumps" directory will be
+rendering the contents of any such pair of files and the difference between
+them (:help term_dumpdiff()); otherwise, they will be rendering own contents
+(:help term_dumpload()).  Remember to execute :edit when occasionally you see
+raw file contents instead of rendered.
+
+At any time, you can add, list, and abandon other screendumps:
+
+    :$argedit dumps/Test_spell_*.dump
+    :args
+    :qall
+
+The listing of argument commands can be found under :help buffer-list.
+
+
+VIEWING GENERATED SCREENDUMPS (from a CI-uploaded artifact):
+
+After you have downloaded an artifact archive containing failed screendumps
+and extracted its files in a temporary directory, you need to set up a "dumps"
+directory by creating a symlink:
+
+    $ cd /path/to/fork
+    $ ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps
+
+You can now examine the extracted screendumps:
+
+    $ ./src/vim -u NONE -S src/testdir/viewdumps.vim \
+				/tmp/src/testdir/failed/*.dump
 
-See the file ../../runtime/syntax/testdir/README.txt section
-"Viewing generated screendumps" on how to analyze failed screen dumps
-(from CI or locally) using the provided Vim script
-../../runtime/syntax/testdir/viewdumps.vim in a more automatic way.
+
+VIEWING GENERATED SCREENDUMPS (submitted for a pull request):
+
+First, you need to check out the topic branch with the proposed changes and
+write down a difference list between the HEAD commit (index) and its parent
+commit with respect to the changed "dumps" filenames:
+
+    $ cd /path/to/fork
+    $ git switch prs/1234
+    $ git diff-index --relative=src/testdir/dumps/ \
+				--name-only prs/1234~1 > /tmp/filelist
+
+Then, you need to check out the master branch, change the current working
+directory to reconcile relative filepaths written in the filenames list,
+possibly create an empty "failed" directory, copy in this directory the old
+"dumps" files, whose names are on the same list, and follow it by checking out
+the topic branch:
+
+    $ git switch master
+    $ cd src/testdir/dumps
+    $ test -d ../failed || mkdir ../failed
+    $ cp -t ../failed $(cat /tmp/filelist)
+    $ git switch prs/1234
+
+Make note of any missing new screendumps.  Please remember about the
+introduced INVERTED relation between "dumps" and "failed", i.e. the files to
+be committed are in "dumps" already and their old versions are in "failed".
+Therefore, you need to copy the missing new screendumps from "dumps" to
+"failed":
+
+    $ cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
+
+After you have changed the current working directory to its parent directory,
+you can now examine the screendumps from the "failed" directory (note that new
+screendumps will be shown with no difference between their versions):
+
+    $ cd ..
+    $ ../vim -u NONE -S viewdumps.vim
+
new file mode 100644
--- /dev/null
+++ b/src/testdir/commondumps.vim
@@ -0,0 +1,76 @@
+vim9script
+
+# (Script-local.)
+#
+# Render a loaded screendump file or the difference of a loaded screendump
+# file and its namesake file from the "dumps" directory.
+def Render()
+  const failed_fname: string = bufname()
+  try
+    setlocal suffixesadd=.dump
+    const dumps_fname: string = findfile(
+			fnamemodify(failed_fname, ':p:t'),
+			fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
+    if filereadable(dumps_fname)
+      term_dumpdiff(failed_fname, dumps_fname)
+    else
+      term_dumpload(failed_fname)
+    endif
+  finally
+    exec 'bwipeout ' .. failed_fname
+  endtry
+enddef
+
+# Search for the "failed" directory in the passed _subtreedirname_ directories
+# (usually "\<src\>" or "\<syntax\>") and, if found, select its passed _count_
+# occurence, add all its "*.dump" files to the argument list and list them;
+# also define a BufRead autocommand that would invoke "Render()" for every
+# "*.dump" file.
+def g:Init(subtreedirname: string, count: number)
+  # Support sourcing this script from any directory in the direct path that
+  # leads to the project's root directory.
+  const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
+    ->filter(((cwdpath: string, parentdirname: string) =>
+			(_: number, dirpath: string) =>
+	cwdpath =~ parentdirname || dirpath =~ parentdirname)(
+			getcwd(),
+			subtreedirname))
+    ->get(count, '') .. '/'
+  var error: string = null_string
+
+  if failed_path == '/'
+    error = 'No such directory: "failed"'
+  else
+    const failed_fnames: string = failed_path .. readdir(failed_path,
+			(fname: string) => fname =~ '^.\+\.dump$')
+      ->join(' ' .. failed_path)
+
+    if failed_fnames =~ 'failed/$'
+      error = 'No such file: "*.dump"'
+    else
+      exec ':0argedit ' .. failed_fnames
+      buffers
+    endif
+  endif
+
+  autocmd_add([{
+    replace:	true,
+    group:	'viewdumps',
+    event:	'BufRead',
+    pattern:	'*.dump',
+    cmd:	'Render()',
+  }])
+
+  # Unconditionally help, in case a list of filenames is passed to the
+  # command, the first terminal window with its BufRead event.
+  silent doautocmd viewdumps BufRead
+
+  if error != null_string
+    # Instead of sleeping, fill half a window with blanks and prompt
+    # hit-enter.
+    echom error .. repeat("\x20",
+	(winwidth(0) * (winheight(0) / 2) - strlen(error)))
+  endif
+enddef
+
+# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
new file mode 100644
--- /dev/null
+++ b/src/testdir/viewdumps.vim
@@ -0,0 +1,11 @@
+vim9script
+
+exec 'source ' .. (((cwdpath: string) => cwdpath
+  ->strpart(0, cwdpath->strridx('/vim')))(getcwd()))
+  .. '/vim/src/testdir/commondumps.vim'
+g:Init('\<src\>', 0)
+
+# Match ":language" of runtest.vim.
+language messages C
+
+# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
--- a/src/version.c
+++ b/src/version.c
@@ -705,6 +705,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    681,
+/**/
     680,
 /**/
     679,