# HG changeset patch # User Bram Moolenaar # Date 1545684305 -3600 # Node ID 544490b69e1d38c677d47330ce7f9b5c1e72ab58 # Parent 1e57afb3e8e9df1382f8cb67a716c3e2181d7418 patch 8.1.0633: crash when out of memory while opening a terminal window commit https://github.com/vim/vim/commit/cd929f7ba8cc5b6d6dcf35c8b34124e969fed6b8 Author: Bram Moolenaar Date: Mon Dec 24 21:38:45 2018 +0100 patch 8.1.0633: crash when out of memory while opening a terminal window Problem: Crash when out of memory while opening a terminal window. Solution: Handle out-of-memory more gracefully. diff --git a/src/libvterm/src/state.c b/src/libvterm/src/state.c --- a/src/libvterm/src/state.c +++ b/src/libvterm/src/state.c @@ -53,6 +53,8 @@ static VTermState *vterm_state_new(VTerm { VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState)); + if (state == NULL) + return NULL; state->vt = vt; state->rows = vt->rows; @@ -1693,6 +1695,10 @@ static const VTermParserCallbacks parser on_resize /* resize */ }; +/* + * Return the existing state or create a new one. + * Returns NULL when out of memory. + */ VTermState *vterm_obtain_state(VTerm *vt) { VTermState *state; @@ -1700,6 +1706,8 @@ VTermState *vterm_obtain_state(VTerm *vt return vt->state; state = vterm_state_new(vt); + if (state == NULL) + return NULL; vt->state = state; state->combine_chars_size = 16; diff --git a/src/libvterm/src/termscreen.c b/src/libvterm/src/termscreen.c --- a/src/libvterm/src/termscreen.c +++ b/src/libvterm/src/termscreen.c @@ -1,5 +1,6 @@ #include "vterm_internal.h" +/* vim: set sw=2 : */ #include #include @@ -95,8 +96,7 @@ static ScreenCell *realloc_buffer(VTermS } } - if(buffer) - vterm_allocator_free(screen->vt, buffer); + vterm_allocator_free(screen->vt, buffer); return new_buffer; } @@ -518,8 +518,7 @@ static int resize(int new_rows, int new_ screen->rows = new_rows; screen->cols = new_cols; - if(screen->sb_buffer) - vterm_allocator_free(screen->vt, screen->sb_buffer); + vterm_allocator_free(screen->vt, screen->sb_buffer); screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols); @@ -619,16 +618,21 @@ static VTermStateCallbacks state_cbs = { &setlineinfo /* setlineinfo */ }; +/* + * Allocate a new screen and return it. + * Return NULL when out of memory. + */ static VTermScreen *screen_new(VTerm *vt) { VTermState *state = vterm_obtain_state(vt); VTermScreen *screen; int rows, cols; - if(!state) + if (state == NULL) return NULL; - screen = vterm_allocator_malloc(vt, sizeof(VTermScreen)); + if (screen == NULL) + return NULL; vterm_get_size(vt, &rows, &cols); @@ -646,10 +650,13 @@ static VTermScreen *screen_new(VTerm *vt screen->cbdata = NULL; screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols); - screen->buffer = screen->buffers[0]; - screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols); + if (screen->buffer == NULL || screen->sb_buffer == NULL) + { + vterm_screen_free(screen); + return NULL; + } vterm_state_set_callbacks(screen->state, &state_cbs, screen); @@ -659,11 +666,8 @@ static VTermScreen *screen_new(VTerm *vt INTERNAL void vterm_screen_free(VTermScreen *screen) { vterm_allocator_free(screen->vt, screen->buffers[0]); - if(screen->buffers[1]) - vterm_allocator_free(screen->vt, screen->buffers[1]); - + vterm_allocator_free(screen->vt, screen->buffers[1]); vterm_allocator_free(screen->vt, screen->sb_buffer); - vterm_allocator_free(screen->vt, screen); } diff --git a/src/libvterm/src/vterm.c b/src/libvterm/src/vterm.c --- a/src/libvterm/src/vterm.c +++ b/src/libvterm/src/vterm.c @@ -1,5 +1,6 @@ #define DEFINE_INLINES +/* vim: set sw=2 : */ #include "vterm_internal.h" #include @@ -41,6 +42,8 @@ VTerm *vterm_new_with_allocator(int rows /* Need to bootstrap using the allocator function directly */ VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata); + if (vt == NULL) + return NULL; vt->allocator = funcs; vt->allocdata = allocdata; @@ -55,10 +58,21 @@ VTerm *vterm_new_with_allocator(int rows vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */ vt->parser.strbuffer_cur = 0; vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len); + if (vt->parser.strbuffer == NULL) + { + vterm_allocator_free(vt, vt); + return NULL; + } vt->outbuffer_len = 200; vt->outbuffer_cur = 0; vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len); + if (vt->outbuffer == NULL) + { + vterm_allocator_free(vt, vt->parser.strbuffer); + vterm_allocator_free(vt, vt); + return NULL; + } return vt; } @@ -82,9 +96,13 @@ INTERNAL void *vterm_allocator_malloc(VT return (*vt->allocator->malloc)(size, vt->allocdata); } +/* + * Free "ptr" unless it is NULL. + */ INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr) { - (*vt->allocator->free)(ptr, vt->allocdata); + if (ptr) + (*vt->allocator->free)(ptr, vt->allocdata); } void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp) diff --git a/src/terminal.c b/src/terminal.c --- a/src/terminal.c +++ b/src/terminal.c @@ -3430,6 +3430,7 @@ set_vterm_palette(VTerm *vterm, long_u * { int index = 0; VTermState *state = vterm_obtain_state(vterm); + for (; index < 16; index++) { VTermColor color; @@ -3703,8 +3704,9 @@ static VTermAllocatorFunctions vterm_all /* * Create a new vterm and initialize it. + * Return FAIL when out of memory. */ - static void + static int create_vterm(term_T *term, int rows, int cols) { VTerm *vterm; @@ -3714,7 +3716,18 @@ create_vterm(term_T *term, int rows, int vterm = vterm_new_with_allocator(rows, cols, &vterm_allocator, NULL); term->tl_vterm = vterm; + if (vterm == NULL) + return FAIL; + + // Allocate screen and state here, so we can bail out if that fails. + state = vterm_obtain_state(vterm); screen = vterm_obtain_screen(vterm); + if (state == NULL || screen == NULL) + { + vterm_free(vterm); + return FAIL; + } + vterm_screen_set_callbacks(screen, &screen_callbacks, term); /* TODO: depends on 'encoding'. */ vterm_set_utf8(vterm, 1); @@ -3722,7 +3735,7 @@ create_vterm(term_T *term, int rows, int init_default_colors(term); vterm_state_set_default_colors( - vterm_obtain_state(vterm), + state, &term->tl_default_color.fg, &term->tl_default_color.bg); @@ -3746,9 +3759,10 @@ create_vterm(term_T *term, int rows, int #else value.boolean = 0; #endif - state = vterm_obtain_state(vterm); vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value); vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term); + + return OK; } /* @@ -5629,7 +5643,8 @@ term_and_job_init( vim_free(cwd_wchar); vim_free(env_wchar); - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + goto failed; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) @@ -5710,7 +5725,8 @@ create_pty_only(term_T *term, jobopt_T * char in_name[80], out_name[80]; channel_T *channel = NULL; - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d", GetCurrentProcessId(), @@ -5822,7 +5838,8 @@ term_and_job_init( jobopt_T *opt, jobopt_T *orig_opt UNUSED) { - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS) if (opt->jo_set2 & JO2_ANSI_COLORS) @@ -5844,7 +5861,8 @@ term_and_job_init( static int create_pty_only(term_T *term, jobopt_T *opt) { - create_vterm(term, term->tl_rows, term->tl_cols); + if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL) + return FAIL; term->tl_job = job_alloc(); if (term->tl_job == NULL) diff --git a/src/version.c b/src/version.c --- a/src/version.c +++ b/src/version.c @@ -800,6 +800,8 @@ static char *(features[]) = static int included_patches[] = { /* Add new patch number below this line */ /**/ + 633, +/**/ 632, /**/ 631,