Mercurial > vim
changeset 15249:544490b69e1d v8.1.0633
patch 8.1.0633: crash when out of memory while opening a terminal window
commit https://github.com/vim/vim/commit/cd929f7ba8cc5b6d6dcf35c8b34124e969fed6b8
Author: Bram Moolenaar <Bram@vim.org>
Date: Mon Dec 24 21:38:45 2018 +0100
patch 8.1.0633: crash when out of memory while opening a terminal window
Problem: Crash when out of memory while opening a terminal window.
Solution: Handle out-of-memory more gracefully.
| author | Bram Moolenaar <Bram@vim.org> |
|---|---|
| date | Mon, 24 Dec 2018 21:45:05 +0100 |
| parents | 1e57afb3e8e9 |
| children | 82208e2fbdf3 |
| files | src/libvterm/src/state.c src/libvterm/src/termscreen.c src/libvterm/src/vterm.c src/terminal.c src/version.c |
| diffstat | 5 files changed, 70 insertions(+), 20 deletions(-) [+] |
line wrap: on
line diff
--- a/src/libvterm/src/state.c +++ b/src/libvterm/src/state.c @@ -53,6 +53,8 @@ static VTermState *vterm_state_new(VTerm { VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState)); + if (state == NULL) + return NULL; state->vt = vt; state->rows = vt->rows; @@ -1693,6 +1695,10 @@ static const VTermParserCallbacks parser on_resize /* resize */ }; +/* + * Return the existing state or create a new one. + * Returns NULL when out of memory. + */ VTermState *vterm_obtain_state(VTerm *vt) { VTermState *state; @@ -1700,6 +1706,8 @@ VTermState *vterm_obtain_state(VTerm *vt return vt->state; state = vterm_state_new(vt); + if (state == NULL) + return NULL; vt->state = state; state->combine_chars_size = 16;
--- a/src/libvterm/src/termscreen.c +++ b/src/libvterm/src/termscreen.c @@ -1,5 +1,6 @@ #include "vterm_internal.h" +/* vim: set sw=2 : */ #include <stdio.h> #include <string.h> @@ -95,8 +96,7 @@ static ScreenCell *realloc_buffer(VTermS } } - if(buffer) - vterm_allocator_free(screen->vt, buffer); + vterm_allocator_free(screen->vt, buffer); return new_buffer; } @@ -518,8 +518,7 @@ static int resize(int new_rows, int new_ screen->rows = new_rows; screen->cols = new_cols; - if(screen->sb_buffer) - vterm_allocator_free(screen->vt, screen->sb_buffer); + vterm_allocator_free(screen->vt, screen->sb_buffer); screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols); @@ -619,16 +618,21 @@ static VTermStateCallbacks state_cbs = { &setlineinfo /* setlineinfo */ }; +/* + * Allocate a new screen and return it. + * Return NULL when out of memory. + */ static VTermScreen *screen_new(VTerm *vt) { VTermState *state = vterm_obtain_state(vt); VTermScreen *screen; int rows, cols; - if(!state) + if (state == NULL) return NULL; - screen = vterm_allocator_malloc(vt, sizeof(VTermScreen)); + if (screen == NULL) + return NULL; vterm_get_size(vt, &rows, &cols); @@ -646,10 +650,13 @@ static VTermScreen *screen_new(VTerm *vt screen->cbdata = NULL; screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols); - screen->buffer = screen->buffers[0]; - screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols); + if (screen->buffer == NULL || screen->sb_buffer == NULL) + { + vterm_screen_free(screen); + return NULL; + } vterm_state_set_callbacks(screen->state, &state_cbs, screen); @@ -659,11 +666,8 @@ static VTermScreen *screen_new(VTerm *vt INTERNAL void vterm_screen_free(VTermScreen *screen) { vterm_allocator_free(screen->vt, screen->buffers[0]); - if(screen->buffers[1]) - vterm_allocator_free(screen->vt, screen->buffers[1]); - + vterm_allocator_free(screen->vt, screen->buffers[1]); vterm_allocator_free(screen->vt, screen->sb_buffer); - vterm_allocator_free(screen->vt, screen); }
--- a/src/libvterm/src/vterm.c +++ b/src/libvterm/src/vterm.c @@ -1,5 +1,6 @@ #define DEFINE_INLINES +/* vim: set sw=2 : */ #include "vterm_internal.h" #include <stdio.h> @@ -41,6 +42,8 @@ VTerm *vterm_new_with_allocator(int rows /* Need to bootstrap using the allocator function directly */ VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata); + if (vt == NULL) + return NULL; vt->allocator = funcs; vt->allocdata = allocdata; @@ -55,10 +58,21 @@ VTerm *vterm_new_with_allocator(int rows vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */ vt->parser.strbuffer_cur = 0; vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len); + if (vt->parser.strbuffer == NULL) + { + vterm_allocator_free(vt, vt); + return NULL; + } vt->outbuffer_len = 200; vt->outbuffer_cur = 0; vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len); + if (vt->outbuffer == NULL) + { + vterm_allocator_free(vt, vt->parser.strbuffer); + vterm_allocator_free(vt, vt); + return NULL; + } return vt; } @@ -82,9 +96,13 @@ INTERNAL void *vterm_allocator_malloc(VT return (*vt->allocator->malloc)(size, vt->allocdata); } +/* + * Free "ptr" unless it is NULL. + */ INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr) { - (*vt->allocator->free)(ptr, vt->allocdata); + if (ptr) + (*vt->allocator->free)(ptr, vt->allocdata); } void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)
--- a/src/terminal.c +++ b/src/terminal.c @@ -3430,6 +3430,7 @@ set_vterm_palette(VTerm *vterm, long_u * { int index = 0; VTermState *state = vterm_obtain_state(vterm); + for (; index < 16; index++) { VTermColor color; @@ -3703,8 +3704,9 @@ static VTermAllocatorFunctions vterm_all /* * Create a new vterm and initialize it. + * Return FAIL when out of memory. */ - static void + static int create_vterm(term_T *term, int rows, int cols) { VTerm *vterm; @@ -3714,7 +3716,18 @@ create_vterm(term_T *term, int rows, int vterm = vterm_new_with_allocator(rows, cols, &vterm_allocator, NULL); term->tl_vterm = vterm; + if (vterm == NULL) + return FAIL; + + // Allocate screen and state here, so we can bail out if that fails. + state = vterm_obtain_state(vterm); screen = vterm_obtain_screen(vterm); + if (state == NULL || screen == NULL) + { + vterm_free(vterm); + return FAIL; + } + vterm_screen_set_callbacks(screen, &screen_callbacks, term); /* TODO: depends on 'encoding'. */ vterm_set_utf8(vterm, 1); @@ -3722,7 +3735,7 @@ create_vterm(term_T *term, int rows, int init_default_colors(term); vterm_state_set_default_colors( - vterm_obtain_state(vterm), + state, &term->tl_default_color.fg, &term->tl_default_color.bg); @@ -3746,9 +3759,10 @@ create_vterm(term_T *term, int rows, int #else value.boolean = 0; #endif - state = vterm_obtain_state(vterm); vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value); vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term); + + return OK; } /* @@ -5629,7 +5643,8 @@ term_and_job_init( vim_free(cwd_wchar); vim_free(env_wchar); - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + goto failed; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) @@ -5710,7 +5725,8 @@ create_pty_only(term_T *term, jobopt_T * char in_name[80], out_name[80]; channel_T *channel = NULL; - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d", GetCurrentProcessId(), @@ -5822,7 +5838,8 @@ term_and_job_init( jobopt_T *opt, jobopt_T *orig_opt UNUSED) { - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) @@ -5844,7 +5861,8 @@ term_and_job_init( static int create_pty_only(term_T *term, jobopt_T *opt) { - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; term->tl_job = job_alloc(); if (term->tl_job == NULL)