Mercurial > vim
changeset 11063:e71d3bdf3bc3 v8.0.0420
patch 8.0.0420: text garbled when the system encoding differs from 'encoding'
commit https://github.com/vim/vim/commit/2c7292dc5bbf155fe2192d417363b8c085759cad
Author: Bram Moolenaar <Bram@vim.org>
Date: Sun Mar 5 17:43:31 2017 +0100
patch 8.0.0420: text garbled when the system encoding differs from 'encoding'
Problem: When running :make the output may be in the system encoding,
different from 'encoding'.
Solution: Add the 'makeencoding' option. (Ken Takata)
| author | Christian Brabandt <cb@256bit.org> |
|---|---|
| date | Sun, 05 Mar 2017 17:45:05 +0100 |
| parents | 1218c5353e2b |
| children | 1901db360ca6 |
| files | runtime/doc/options.txt runtime/doc/quickfix.txt runtime/doc/quickref.txt src/Makefile src/buffer.c src/if_cscope.c src/main.c src/option.c src/option.h src/proto/quickfix.pro src/quickfix.c src/structs.h src/testdir/Make_all.mak src/testdir/test_makeencoding.py src/testdir/test_makeencoding.vim src/version.c |
| diffstat | 16 files changed, 328 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/runtime/doc/options.txt +++ b/runtime/doc/options.txt @@ -4994,6 +4994,25 @@ A jump table for the options with a shor This option cannot be set from a |modeline| or in the |sandbox|, for security reasons. + *'makeencoding'* *'menc'* +'makeencoding' 'menc' string (default "") + global or local to buffer |global-local| + {only available when compiled with the |+multi_byte| + feature} + {not in Vi} + Encoding used for reading the output of external commands. When empty, + encoding is not converted. + This is used for `:make`, `:lmake`, `:grep`, `:lgrep`, `:grepadd`, + `:lgrepadd`, `:cfile`, `:cgetfile`, `:caddfile`, `:lfile`, `:lgetfile`, + and `:laddfile`. + + This would be mostly useful when you use MS-Windows and set 'encoding' + to "utf-8". If |+iconv| is enabled and GNU libiconv is used, setting + 'makeencoding' to "char" has the same effect as setting to the system + locale encoding. Example: > + :set encoding=utf-8 + :set makeencoding=char " system locale is used +< *'makeprg'* *'mp'* 'makeprg' 'mp' string (default "make", VMS: "MMS") global or local to buffer |global-local|
--- a/runtime/doc/quickfix.txt +++ b/runtime/doc/quickfix.txt @@ -164,6 +164,9 @@ processing a quickfix or location list c keep Vim running while compiling. If you give the name of the errorfile, the 'errorfile' option will be set to [errorfile]. See |:cc| for [!]. + If the encoding of the error file differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. *:lf* *:lfile* :lf[ile][!] [errorfile] Same as ":cfile", except the location list for the @@ -175,6 +178,9 @@ processing a quickfix or location list c :cg[etfile] [errorfile] *:cg* *:cgetfile* Read the error file. Just like ":cfile" but don't jump to the first error. + If the encoding of the error file differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. :lg[etfile] [errorfile] *:lg* *:lgetfile* @@ -185,6 +191,9 @@ processing a quickfix or location list c :caddf[ile] [errorfile] Read the error file and add the errors from the errorfile to the current quickfix list. If a quickfix list is not present, then a new list is created. + If the encoding of the error file differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. *:laddf* *:laddfile* :laddf[ile] [errorfile] Same as ":caddfile", except the location list for the @@ -320,6 +329,7 @@ use this code: > endfunction au QuickfixCmdPost make call QfMakeConv() +Another option is using 'makeencoding'. EXECUTE A COMMAND IN ALL THE BUFFERS IN QUICKFIX OR LOCATION LIST: @@ -586,6 +596,9 @@ 4. Using :make *:make_makeprg* like |:cnext| and |:cprevious|, see above. This command does not accept a comment, any " characters are considered part of the arguments. + If the encoding of the program output differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. *:lmak* *:lmake* :lmak[e][!] [arguments] @@ -645,6 +658,7 @@ read the error messages: > au QuickfixCmdPost make call QfMakeConv() (Example by Faque Cheng) +Another option is using 'makeencoding'. ============================================================================== 5. Using :vimgrep and :grep *grep* *lid* @@ -759,6 +773,9 @@ id-utils) in a similar way to its compil When 'grepprg' is "internal" this works like |:vimgrep|. Note that the pattern needs to be enclosed in separator characters then. + If the encoding of the program output differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. *:lgr* *:lgrep* :lgr[ep][!] [arguments] Same as ":grep", except the location list for the @@ -783,6 +800,10 @@ id-utils) in a similar way to its compil \ | catch /E480:/ \ | endtry" < + If the encoding of the program output differs from the + 'encoding' option, you can use the 'makeencoding' + option to specify the encoding. + *:lgrepa* *:lgrepadd* :lgrepa[dd][!] [arguments] Same as ":grepadd", except the location list for the
--- a/runtime/doc/quickref.txt +++ b/runtime/doc/quickref.txt @@ -782,6 +782,7 @@ Short explanation of each option: *opti 'macatsui' Mac GUI: use ATSUI text drawing 'magic' changes special characters in search patterns 'makeef' 'mef' name of the errorfile for ":make" +'makeencoding' 'menc' encoding of external make/grep commands 'makeprg' 'mp' program to use for the ":make" command 'matchpairs' 'mps' pairs of characters that "%" can match 'matchtime' 'mat' tenths of a second to show matching paren
--- a/src/Makefile +++ b/src/Makefile @@ -2157,6 +2157,7 @@ test_arglist \ test_listlbr \ test_listlbr_utf8 \ test_lua \ + test_makeencoding \ test_man \ test_mapping \ test_marks \
--- a/src/buffer.c +++ b/src/buffer.c @@ -2255,6 +2255,9 @@ free_buf_options( clear_string_option(&buf->b_p_lw); #endif clear_string_option(&buf->b_p_bkc); +#ifdef FEAT_MBYTE + clear_string_option(&buf->b_p_menc); +#endif } /*
--- a/src/if_cscope.c +++ b/src/if_cscope.c @@ -1242,7 +1242,7 @@ cs_find_common( wp = curwin; /* '-' starts a new error list */ if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m", - *qfpos == '-', cmdline) > 0) + *qfpos == '-', cmdline, NULL) > 0) { # ifdef FEAT_WINDOWS if (postponed_split != 0)
--- a/src/main.c +++ b/src/main.c @@ -559,11 +559,16 @@ vim_main2(void) */ if (params.edit_type == EDIT_QF) { + char_u *enc = NULL; + +# ifdef FEAT_MBYTE + enc = p_menc; +# endif if (params.use_ef != NULL) set_string_option_direct((char_u *)"ef", -1, params.use_ef, OPT_FREE, SID_CARG); vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef); - if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff) < 0) + if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff, enc) < 0) { out_char('\n'); mch_exit(3);
--- a/src/option.c +++ b/src/option.c @@ -139,6 +139,9 @@ # define PV_LISP OPT_BUF(BV_LISP) # define PV_LW OPT_BOTH(OPT_BUF(BV_LW)) #endif +#ifdef FEAT_MBYTE +# define PV_MENC OPT_BOTH(OPT_BUF(BV_MENC)) +#endif #define PV_MA OPT_BUF(BV_MA) #define PV_ML OPT_BUF(BV_ML) #define PV_MOD OPT_BUF(BV_MOD) @@ -1900,6 +1903,15 @@ static struct vimoption options[] = {(char_u *)NULL, (char_u *)0L} #endif SCRIPTID_INIT}, + {"makeencoding","menc", P_STRING|P_VI_DEF, +#ifdef FEAT_MBYTE + (char_u *)&p_menc, PV_MENC, + {(char_u *)"", (char_u *)0L} +#else + (char_u *)NULL, PV_NONE, + {(char_u *)0L, (char_u *)0L} +#endif + SCRIPTID_INIT}, {"makeprg", "mp", P_STRING|P_EXPAND|P_VI_DEF|P_SECURE, #ifdef FEAT_QUICKFIX (char_u *)&p_mp, PV_MP, @@ -5686,6 +5698,9 @@ check_buf_options(buf_T *buf) check_string_option(&buf->b_p_lw); #endif check_string_option(&buf->b_p_bkc); +#ifdef FEAT_MBYTE + check_string_option(&buf->b_p_menc); +#endif } /* @@ -6289,8 +6304,9 @@ did_set_string_option( #endif #ifdef FEAT_MBYTE - /* 'encoding' and 'fileencoding' */ - else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc) + /* 'encoding', 'fileencoding', 'termencoding' and 'makeencoding' */ + else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc + || gvarp == &p_menc) { if (gvarp == &p_fenc) { @@ -10425,6 +10441,11 @@ unset_global_local_option(char_u *name, clear_string_option(&buf->b_p_lw); break; #endif +#ifdef FEAT_MBYTE + case PV_MENC: + clear_string_option(&buf->b_p_menc); + break; +#endif } } @@ -10478,6 +10499,9 @@ get_varp_scope(struct vimoption *p, int case PV_LW: return (char_u *)&(curbuf->b_p_lw); #endif case PV_BKC: return (char_u *)&(curbuf->b_p_bkc); +#ifdef FEAT_MBYTE + case PV_MENC: return (char_u *)&(curbuf->b_p_menc); +#endif } return NULL; /* "cannot happen" */ } @@ -10553,6 +10577,10 @@ get_varp(struct vimoption *p) case PV_LW: return *curbuf->b_p_lw != NUL ? (char_u *)&(curbuf->b_p_lw) : p->var; #endif +#ifdef FEAT_MBYTE + case PV_MENC: return *curbuf->b_p_menc != NUL + ? (char_u *)&(curbuf->b_p_menc) : p->var; +#endif #ifdef FEAT_ARABIC case PV_ARAB: return (char_u *)&(curwin->w_p_arab); @@ -11154,6 +11182,9 @@ buf_copy_options(buf_T *buf, int flags) #ifdef FEAT_LISP buf->b_p_lw = empty_option; #endif +#ifdef FEAT_MBYTE + buf->b_p_menc = empty_option; +#endif /* * Don't copy the options set by ex_help(), use the saved values,
--- a/src/option.h +++ b/src/option.h @@ -630,6 +630,9 @@ EXTERN char_u *p_luadll; /* 'luadll' */ EXTERN int p_macatsui; /* 'macatsui' */ #endif EXTERN int p_magic; /* 'magic' */ +#ifdef FEAT_MBYTE +EXTERN char_u *p_menc; /* 'makeencoding' */ +#endif #ifdef FEAT_QUICKFIX EXTERN char_u *p_mef; /* 'makeef' */ EXTERN char_u *p_mp; /* 'makeprg' */ @@ -1065,6 +1068,9 @@ enum , BV_LISP , BV_LW #endif +#ifdef FEAT_MBYTE + , BV_MENC +#endif , BV_MA , BV_ML , BV_MOD
--- a/src/proto/quickfix.pro +++ b/src/proto/quickfix.pro @@ -1,5 +1,5 @@ /* quickfix.c */ -int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title); +int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title, char_u *enc); void qf_free_all(win_T *wp); void copy_loclist(win_T *from, win_T *to); void qf_jump(qf_info_T *qi, int dir, int errornr, int forceit);
--- a/src/quickfix.c +++ b/src/quickfix.c @@ -116,7 +116,7 @@ struct efm_S static efm_T *fmt_start = NULL; /* cached across qf_parse_line() calls */ -static int qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title); +static int qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title, char_u *enc); static void qf_store_title(qf_info_T *qi, char_u *title); static void qf_new_list(qf_info_T *qi, char_u *qf_title); static void ll_free_all(qf_info_T **pqi); @@ -167,7 +167,8 @@ qf_init( char_u *efile, char_u *errorformat, int newlist, /* TRUE: start a new error list */ - char_u *qf_title) + char_u *qf_title, + char_u *enc) { qf_info_T *qi = &ql_info; @@ -180,7 +181,7 @@ qf_init( return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist, (linenr_T)0, (linenr_T)0, - qf_title); + qf_title, enc); } /* @@ -498,6 +499,7 @@ typedef struct { buf_T *buf; linenr_T buflnum; linenr_T lnumlast; + vimconv_T vc; } qfstate_T; static char_u * @@ -713,6 +715,30 @@ qf_get_next_file_line(qfstate_T *state) else state->linebuf = IObuff; +#ifdef FEAT_MBYTE + /* Convert a line if it contains a non-ASCII character. */ + if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) { + char_u *line; + + line = string_convert(&state->vc, state->linebuf, &state->linelen); + if (line != NULL) + { + if (state->linelen < IOSIZE) + { + STRCPY(state->linebuf, line); + vim_free(line); + } + else + { + vim_free(state->growbuf); + state->linebuf = state->growbuf = line; + state->growbufsiz = state->linelen < LINE_MAXLEN + ? state->linelen : LINE_MAXLEN; + } + } + } +#endif + return QF_OK; } @@ -1105,11 +1131,11 @@ qf_init_ext( int newlist, /* TRUE: start a new error list */ linenr_T lnumfirst, /* first line number to use */ linenr_T lnumlast, /* last line number to use */ - char_u *qf_title) + char_u *qf_title, + char_u *enc) { - qfstate_T state = {NULL, 0, NULL, 0, NULL, NULL, NULL, NULL, - NULL, 0, 0}; - qffields_T fields = {NULL, NULL, 0, 0L, 0, FALSE, NULL, 0, 0, 0}; + qfstate_T state; + qffields_T fields; #ifdef FEAT_WINDOWS qfline_T *old_last = NULL; int adding = FALSE; @@ -1120,6 +1146,13 @@ qf_init_ext( int retval = -1; /* default: return error flag */ int status; + vim_memset(&state, 0, sizeof(state)); + vim_memset(&fields, 0, sizeof(fields)); +#ifdef FEAT_MBYTE + state.vc.vc_type = CONV_NONE; + if (enc != NULL && *enc != NUL) + convert_setup(&state.vc, enc, p_enc); +#endif fields.namebuf = alloc_id(CMDBUFFSIZE + 1, aid_qf_namebuf); fields.errmsglen = CMDBUFFSIZE + 1; fields.errmsg = alloc_id(fields.errmsglen, aid_qf_errmsg); @@ -1286,6 +1319,10 @@ qf_init_end: #ifdef FEAT_WINDOWS qf_update_buffer(qi, old_last); #endif +#ifdef FEAT_MBYTE + if (state.vc.vc_type != CONV_NONE) + convert_setup(&state.vc, NULL, NULL); +#endif return retval; } @@ -3431,6 +3468,7 @@ ex_make(exarg_T *eap) { char_u *fname; char_u *cmd; + char_u *enc = NULL; unsigned len; win_T *wp = NULL; qf_info_T *qi = &ql_info; @@ -3464,6 +3502,9 @@ ex_make(exarg_T *eap) # endif } #endif +#ifdef FEAT_MBYTE + enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc; +#endif if (eap->cmdidx == CMD_lmake || eap->cmdidx == CMD_lgrep || eap->cmdidx == CMD_lgrepadd) @@ -3511,7 +3552,7 @@ ex_make(exarg_T *eap) && eap->cmdidx != CMD_lmake) ? p_gefm : p_efm, (eap->cmdidx != CMD_grepadd && eap->cmdidx != CMD_lgrepadd), - *eap->cmdlinep); + *eap->cmdlinep, enc); if (wp != NULL) qi = GET_LOC_LIST(wp); #ifdef FEAT_AUTOCMD @@ -3850,6 +3891,7 @@ ex_cnext(exarg_T *eap) void ex_cfile(exarg_T *eap) { + char_u *enc = NULL; win_T *wp = NULL; qf_info_T *qi = &ql_info; #ifdef FEAT_AUTOCMD @@ -3874,6 +3916,9 @@ ex_cfile(exarg_T *eap) if (au_name != NULL) apply_autocmds(EVENT_QUICKFIXCMDPRE, au_name, NULL, FALSE, curbuf); #endif +#ifdef FEAT_MBYTE + enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc; +#endif #ifdef FEAT_BROWSE if (cmdmod.browse) { @@ -3901,7 +3946,7 @@ ex_cfile(exarg_T *eap) */ if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile && eap->cmdidx != CMD_laddfile), - *eap->cmdlinep) > 0 + *eap->cmdlinep, enc) > 0 && (eap->cmdidx == CMD_cfile || eap->cmdidx == CMD_lfile)) { @@ -4927,7 +4972,7 @@ ex_cbuffer(exarg_T *eap) (eap->cmdidx != CMD_caddbuffer && eap->cmdidx != CMD_laddbuffer), eap->line1, eap->line2, - qf_title) > 0) + qf_title, NULL) > 0) { #ifdef FEAT_AUTOCMD if (au_name != NULL) @@ -4995,7 +5040,8 @@ ex_cexpr(exarg_T *eap) if (qf_init_ext(qi, NULL, NULL, tv, p_efm, (eap->cmdidx != CMD_caddexpr && eap->cmdidx != CMD_laddexpr), - (linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0) + (linenr_T)0, (linenr_T)0, *eap->cmdlinep, + NULL) > 0) { #ifdef FEAT_AUTOCMD if (au_name != NULL)
--- a/src/structs.h +++ b/src/structs.h @@ -2133,6 +2133,9 @@ struct file_buffer #ifdef FEAT_LISP int b_p_lisp; /* 'lisp' */ #endif +#ifdef FEAT_MBYTE + char_u *b_p_menc; /* 'makeencoding' */ +#endif char_u *b_p_mps; /* 'matchpairs' */ int b_p_ml; /* 'modeline' */ int b_p_ml_nobin; /* b_p_ml saved for binary mode */
--- a/src/testdir/Make_all.mak +++ b/src/testdir/Make_all.mak @@ -170,6 +170,7 @@ NEW_TESTS = test_arabic.res \ test_listlbr.res \ test_listlbr_utf8.res \ test_lua.res \ + test_makeencoding.res \ test_man.res \ test_marks.res \ test_matchadd_conceal.res \
new file mode 100644 --- /dev/null +++ b/src/testdir/test_makeencoding.py @@ -0,0 +1,67 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Test program for :make, :grep and :cgetfile. + +from __future__ import print_function, unicode_literals +import locale +import io +import sys + +def set_output_encoding(enc=None): + """Set the encoding of stdout and stderr + + arguments: + enc -- Encoding name. + If omitted, locale.getpreferredencoding() is used. + """ + if enc is None: + enc = locale.getpreferredencoding() + + def get_text_writer(fo, **kwargs): + kw = dict(kwargs) + kw.setdefault('errors', 'backslashreplace') # use \uXXXX style + kw.setdefault('closefd', False) + + if sys.version_info[0] < 3: + # Work around for Python 2.x + # New line conversion isn't needed here. Done in somewhere else. + writer = io.open(fo.fileno(), mode='w', newline='', **kw) + write = writer.write # save the original write() function + enc = locale.getpreferredencoding() + def convwrite(s): + if isinstance(s, bytes): + write(s.decode(enc)) # convert to unistr + else: + write(s) + try: + writer.flush() # needed on Windows + except IOError: + pass + writer.write = convwrite + else: + writer = io.open(fo.fileno(), mode='w', **kw) + return writer + + sys.stdout = get_text_writer(sys.stdout, encoding=enc) + sys.stderr = get_text_writer(sys.stderr, encoding=enc) + + +def main(): + enc = 'utf-8' + if len(sys.argv) > 1: + enc = sys.argv[1] + set_output_encoding(enc) + + message_tbl = { + 'utf-8': 'ÀÈÌÒÙ こんにちは 你好', + 'latin1': 'ÀÈÌÒÙ', + 'cp932': 'こんにちは', + 'cp936': '你好', + } + + print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc)) + + +if __name__ == "__main__": + main()
new file mode 100644 --- /dev/null +++ b/src/testdir/test_makeencoding.vim @@ -0,0 +1,106 @@ +" Tests for 'makeencoding'. +if !has('multi_byte') + finish +endif + +source shared.vim + +let s:python = PythonProg() +if s:python == '' + " Can't run this test. + finish +endif + +let s:script = 'test_makeencoding.py' + +let s:message_tbl = { + \ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好', + \ 'latin1': 'ÀÈÌÒÙ', + \ 'cp932': 'こんにちは', + \ 'cp936': '你好', + \} + + +" Tests for :cgetfile and :lgetfile. +func Test_getfile() + set errorfile=Xerror.txt + set errorformat=%f(%l)\ :\ %m + + " :cgetfile + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile + cgetfile + copen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + cclose + endfor + + " :lgetfile + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile + lgetfile + lopen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + lclose + endfor + + call delete(&errorfile) +endfunc + + +" Tests for :grep and :lgrep. +func Test_grep() + let &grepprg = s:python + set grepformat=%f(%l)\ :\ %m + + " :grep + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent grep! " . s:script . " " . enc + copen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + cclose + endfor + + " :lgrep + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent lgrep! " . s:script . " " . enc + lopen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + lclose + endfor +endfunc + + +" Tests for :make and :lmake. +func Test_make() + let &makeprg = s:python + set errorformat=%f(%l)\ :\ %m + + " :make + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent make! " . s:script . " " . enc + copen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + cclose + endfor + + " :lmake + for enc in keys(s:message_tbl) + let &makeencoding = enc + exec "silent lmake! " . s:script . " " . enc + lopen + call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")", + \ getline('.')) + lclose + endfor +endfunc