Mercurial > vim
view src/regexp_nfa.c @ 24970:7e9e53a0368f v8.2.3022
patch 8.2.3022: available encryption methods are not strong enough
Commit: https://github.com/vim/vim/commit/f573c6e1ed58d46d694c802eaf5ae3662a952744
Author: Christian Brabandt <cb@256bit.org>
Date: Sun Jun 20 14:02:16 2021 +0200
patch 8.2.3022: available encryption methods are not strong enough
Problem: Available encryption methods are not strong enough.
Solution: Add initial support for xchaha20. (Christian Brabandt,
closes #8394)
| author | Bram Moolenaar <Bram@vim.org> |
|---|---|
| date | Sun, 20 Jun 2021 14:15:07 +0200 |
| parents | 97789fcef0cf |
| children | 10b269321459 |
line wrap: on
line source
/* vi:set ts=8 sts=4 sw=4 noet: * * NFA regular expression implementation. * * This file is included in "regexp.c". */ /* * Logging of NFA engine. * * The NFA engine can write four log files: * - Error log: Contains NFA engine's fatal errors. * - Dump log: Contains compiled NFA state machine's information. * - Run log: Contains information of matching procedure. * - Debug log: Contains detailed information of matching procedure. Can be * disabled by undefining NFA_REGEXP_DEBUG_LOG. * The first one can also be used without debug mode. * The last three are enabled when compiled as debug mode and individually * disabled by commenting them out. * The log files can get quite big! * Do disable all of this when compiling Vim for debugging, undefine DEBUG in * regexp.c */ #ifdef DEBUG # define NFA_REGEXP_ERROR_LOG "nfa_regexp_error.log" # define ENABLE_LOG # define NFA_REGEXP_DUMP_LOG "nfa_regexp_dump.log" # define NFA_REGEXP_RUN_LOG "nfa_regexp_run.log" # define NFA_REGEXP_DEBUG_LOG "nfa_regexp_debug.log" #endif // Added to NFA_ANY - NFA_NUPPER_IC to include a NL. #define NFA_ADD_NL 31 enum { NFA_SPLIT = -1024, NFA_MATCH, NFA_EMPTY, // matches 0-length NFA_START_COLL, // [abc] start NFA_END_COLL, // [abc] end NFA_START_NEG_COLL, // [^abc] start NFA_END_NEG_COLL, // [^abc] end (postfix only) NFA_RANGE, // range of the two previous items // (postfix only) NFA_RANGE_MIN, // low end of a range NFA_RANGE_MAX, // high end of a range NFA_CONCAT, // concatenate two previous items (postfix // only) NFA_OR, // \| (postfix only) NFA_STAR, // greedy * (postfix only) NFA_STAR_NONGREEDY, // non-greedy * (postfix only) NFA_QUEST, // greedy \? (postfix only) NFA_QUEST_NONGREEDY, // non-greedy \? (postfix only) NFA_BOL, // ^ Begin line NFA_EOL, // $ End line NFA_BOW, // \< Begin word NFA_EOW, // \> End word NFA_BOF, // \%^ Begin file NFA_EOF, // \%$ End file NFA_NEWL, NFA_ZSTART, // Used for \zs NFA_ZEND, // Used for \ze NFA_NOPEN, // Start of subexpression marked with \%( NFA_NCLOSE, // End of subexpr. marked with \%( ... \) NFA_START_INVISIBLE, NFA_START_INVISIBLE_FIRST, NFA_START_INVISIBLE_NEG, NFA_START_INVISIBLE_NEG_FIRST, NFA_START_INVISIBLE_BEFORE, NFA_START_INVISIBLE_BEFORE_FIRST, NFA_START_INVISIBLE_BEFORE_NEG, NFA_START_INVISIBLE_BEFORE_NEG_FIRST, NFA_START_PATTERN, NFA_END_INVISIBLE, NFA_END_INVISIBLE_NEG, NFA_END_PATTERN, NFA_COMPOSING, // Next nodes in NFA are part of the // composing multibyte char NFA_END_COMPOSING, // End of a composing char in the NFA NFA_ANY_COMPOSING, // \%C: Any composing characters. NFA_OPT_CHARS, // \%[abc] // The following are used only in the postfix form, not in the NFA NFA_PREV_ATOM_NO_WIDTH, // Used for \@= NFA_PREV_ATOM_NO_WIDTH_NEG, // Used for \@! NFA_PREV_ATOM_JUST_BEFORE, // Used for \@<= NFA_PREV_ATOM_JUST_BEFORE_NEG, // Used for \@<! NFA_PREV_ATOM_LIKE_PATTERN, // Used for \@> NFA_BACKREF1, // \1 NFA_BACKREF2, // \2 NFA_BACKREF3, // \3 NFA_BACKREF4, // \4 NFA_BACKREF5, // \5 NFA_BACKREF6, // \6 NFA_BACKREF7, // \7 NFA_BACKREF8, // \8 NFA_BACKREF9, // \9 #ifdef FEAT_SYN_HL NFA_ZREF1, // \z1 NFA_ZREF2, // \z2 NFA_ZREF3, // \z3 NFA_ZREF4, // \z4 NFA_ZREF5, // \z5 NFA_ZREF6, // \z6 NFA_ZREF7, // \z7 NFA_ZREF8, // \z8 NFA_ZREF9, // \z9 #endif NFA_SKIP, // Skip characters NFA_MOPEN, NFA_MOPEN1, NFA_MOPEN2, NFA_MOPEN3, NFA_MOPEN4, NFA_MOPEN5, NFA_MOPEN6, NFA_MOPEN7, NFA_MOPEN8, NFA_MOPEN9, NFA_MCLOSE, NFA_MCLOSE1, NFA_MCLOSE2, NFA_MCLOSE3, NFA_MCLOSE4, NFA_MCLOSE5, NFA_MCLOSE6, NFA_MCLOSE7, NFA_MCLOSE8, NFA_MCLOSE9, #ifdef FEAT_SYN_HL NFA_ZOPEN, NFA_ZOPEN1, NFA_ZOPEN2, NFA_ZOPEN3, NFA_ZOPEN4, NFA_ZOPEN5, NFA_ZOPEN6, NFA_ZOPEN7, NFA_ZOPEN8, NFA_ZOPEN9, NFA_ZCLOSE, NFA_ZCLOSE1, NFA_ZCLOSE2, NFA_ZCLOSE3, NFA_ZCLOSE4, NFA_ZCLOSE5, NFA_ZCLOSE6, NFA_ZCLOSE7, NFA_ZCLOSE8, NFA_ZCLOSE9, #endif // NFA_FIRST_NL NFA_ANY, // Match any one character. NFA_IDENT, // Match identifier char NFA_SIDENT, // Match identifier char but no digit NFA_KWORD, // Match keyword char NFA_SKWORD, // Match word char but no digit NFA_FNAME, // Match file name char NFA_SFNAME, // Match file name char but no digit NFA_PRINT, // Match printable char NFA_SPRINT, // Match printable char but no digit NFA_WHITE, // Match whitespace char NFA_NWHITE, // Match non-whitespace char NFA_DIGIT, // Match digit char NFA_NDIGIT, // Match non-digit char NFA_HEX, // Match hex char NFA_NHEX, // Match non-hex char NFA_OCTAL, // Match octal char NFA_NOCTAL, // Match non-octal char NFA_WORD, // Match word char NFA_NWORD, // Match non-word char NFA_HEAD, // Match head char NFA_NHEAD, // Match non-head char NFA_ALPHA, // Match alpha char NFA_NALPHA, // Match non-alpha char NFA_LOWER, // Match lowercase char NFA_NLOWER, // Match non-lowercase char NFA_UPPER, // Match uppercase char NFA_NUPPER, // Match non-uppercase char NFA_LOWER_IC, // Match [a-z] NFA_NLOWER_IC, // Match [^a-z] NFA_UPPER_IC, // Match [A-Z] NFA_NUPPER_IC, // Match [^A-Z] NFA_FIRST_NL = NFA_ANY + NFA_ADD_NL, NFA_LAST_NL = NFA_NUPPER_IC + NFA_ADD_NL, NFA_CURSOR, // Match cursor pos NFA_LNUM, // Match line number NFA_LNUM_GT, // Match > line number NFA_LNUM_LT, // Match < line number NFA_COL, // Match cursor column NFA_COL_GT, // Match > cursor column NFA_COL_LT, // Match < cursor column NFA_VCOL, // Match cursor virtual column NFA_VCOL_GT, // Match > cursor virtual column NFA_VCOL_LT, // Match < cursor virtual column NFA_MARK, // Match mark NFA_MARK_GT, // Match > mark NFA_MARK_LT, // Match < mark NFA_VISUAL, // Match Visual area // Character classes [:alnum:] etc NFA_CLASS_ALNUM, NFA_CLASS_ALPHA, NFA_CLASS_BLANK, NFA_CLASS_CNTRL, NFA_CLASS_DIGIT, NFA_CLASS_GRAPH, NFA_CLASS_LOWER, NFA_CLASS_PRINT, NFA_CLASS_PUNCT, NFA_CLASS_SPACE, NFA_CLASS_UPPER, NFA_CLASS_XDIGIT, NFA_CLASS_TAB, NFA_CLASS_RETURN, NFA_CLASS_BACKSPACE, NFA_CLASS_ESCAPE, NFA_CLASS_IDENT, NFA_CLASS_KEYWORD, NFA_CLASS_FNAME }; // Keep in sync with classchars. static int nfa_classcodes[] = { NFA_ANY, NFA_IDENT, NFA_SIDENT, NFA_KWORD,NFA_SKWORD, NFA_FNAME, NFA_SFNAME, NFA_PRINT, NFA_SPRINT, NFA_WHITE, NFA_NWHITE, NFA_DIGIT, NFA_NDIGIT, NFA_HEX, NFA_NHEX, NFA_OCTAL, NFA_NOCTAL, NFA_WORD, NFA_NWORD, NFA_HEAD, NFA_NHEAD, NFA_ALPHA, NFA_NALPHA, NFA_LOWER, NFA_NLOWER, NFA_UPPER, NFA_NUPPER }; static char_u e_nul_found[] = N_("E865: (NFA) Regexp end encountered prematurely"); static char_u e_misplaced[] = N_("E866: (NFA regexp) Misplaced %c"); static char_u e_ill_char_class[] = N_("E877: (NFA regexp) Invalid character class: %d"); static char_u e_value_too_large[] = N_("E951: \\% value too large"); // Variables only used in nfa_regcomp() and descendants. static int nfa_re_flags; // re_flags passed to nfa_regcomp() static int *post_start; // holds the postfix form of r.e. static int *post_end; static int *post_ptr; // Set when the pattern should use the NFA engine. // E.g. [[:upper:]] only allows 8bit characters for BT engine, // while NFA engine handles multibyte characters correctly. static int wants_nfa; static int nstate; // Number of states in the NFA. static int istate; // Index in the state vector, used in alloc_state() // If not NULL match must end at this position static save_se_T *nfa_endp = NULL; // 0 for first call to nfa_regmatch(), 1 for recursive call. static int nfa_ll_index = 0; static int realloc_post_list(void); static int nfa_reg(int paren); #ifdef DEBUG static void nfa_print_state2(FILE *debugf, nfa_state_T *state, garray_T *indent); #endif static int match_follows(nfa_state_T *startstate, int depth); static int failure_chance(nfa_state_T *state, int depth); // helper functions used when doing re2post() ... regatom() parsing #define EMIT(c) do { \ if (post_ptr >= post_end && realloc_post_list() == FAIL) \ return FAIL; \ *post_ptr++ = c; \ } while (0) /* * Initialize internal variables before NFA compilation. * Return OK on success, FAIL otherwise. */ static int nfa_regcomp_start( char_u *expr, int re_flags) // see vim_regcomp() { size_t postfix_size; int nstate_max; nstate = 0; istate = 0; // A reasonable estimation for maximum size nstate_max = (int)(STRLEN(expr) + 1) * 25; // Some items blow up in size, such as [A-z]. Add more space for that. // When it is still not enough realloc_post_list() will be used. nstate_max += 1000; // Size for postfix representation of expr. postfix_size = sizeof(int) * nstate_max; post_start = alloc(postfix_size); if (post_start == NULL) return FAIL; post_ptr = post_start; post_end = post_start + nstate_max; wants_nfa = FALSE; rex.nfa_has_zend = FALSE; rex.nfa_has_backref = FALSE; // shared with BT engine regcomp_start(expr, re_flags); return OK; } /* * Figure out if the NFA state list starts with an anchor, must match at start * of the line. */ static int nfa_get_reganch(nfa_state_T *start, int depth) { nfa_state_T *p = start; if (depth > 4) return 0; while (p != NULL) { switch (p->c) { case NFA_BOL: case NFA_BOF: return 1; // yes! case NFA_ZSTART: case NFA_ZEND: case NFA_CURSOR: case NFA_VISUAL: case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: case NFA_NOPEN: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif p = p->out; break; case NFA_SPLIT: return nfa_get_reganch(p->out, depth + 1) && nfa_get_reganch(p->out1, depth + 1); default: return 0; // noooo } } return 0; } /* * Figure out if the NFA state list starts with a character which must match * at start of the match. */ static int nfa_get_regstart(nfa_state_T *start, int depth) { nfa_state_T *p = start; if (depth > 4) return 0; while (p != NULL) { switch (p->c) { // all kinds of zero-width matches case NFA_BOL: case NFA_BOF: case NFA_BOW: case NFA_EOW: case NFA_ZSTART: case NFA_ZEND: case NFA_CURSOR: case NFA_VISUAL: case NFA_LNUM: case NFA_LNUM_GT: case NFA_LNUM_LT: case NFA_COL: case NFA_COL_GT: case NFA_COL_LT: case NFA_VCOL: case NFA_VCOL_GT: case NFA_VCOL_LT: case NFA_MARK: case NFA_MARK_GT: case NFA_MARK_LT: case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: case NFA_NOPEN: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif p = p->out; break; case NFA_SPLIT: { int c1 = nfa_get_regstart(p->out, depth + 1); int c2 = nfa_get_regstart(p->out1, depth + 1); if (c1 == c2) return c1; // yes! return 0; } default: if (p->c > 0) return p->c; // yes! return 0; } } return 0; } /* * Figure out if the NFA state list contains just literal text and nothing * else. If so return a string in allocated memory with what must match after * regstart. Otherwise return NULL. */ static char_u * nfa_get_match_text(nfa_state_T *start) { nfa_state_T *p = start; int len = 0; char_u *ret; char_u *s; if (p->c != NFA_MOPEN) return NULL; // just in case p = p->out; while (p->c > 0) { len += MB_CHAR2LEN(p->c); p = p->out; } if (p->c != NFA_MCLOSE || p->out->c != NFA_MATCH) return NULL; ret = alloc(len); if (ret != NULL) { p = start->out->out; // skip first char, it goes into regstart s = ret; while (p->c > 0) { if (has_mbyte) s += (*mb_char2bytes)(p->c, s); else *s++ = p->c; p = p->out; } *s = NUL; } return ret; } /* * Allocate more space for post_start. Called when * running above the estimated number of states. */ static int realloc_post_list(void) { int nstate_max = (int)(post_end - post_start); int new_max; int *new_start; int *old_start; // For weird patterns the number of states can be very high. Increasing by // 50% seems a reasonable compromise between memory use and speed. new_max = nstate_max * 3 / 2; new_start = ALLOC_MULT(int, new_max); if (new_start == NULL) return FAIL; mch_memmove(new_start, post_start, nstate_max * sizeof(int)); old_start = post_start; post_start = new_start; post_ptr = new_start + (post_ptr - old_start); post_end = post_start + new_max; vim_free(old_start); return OK; } /* * Search between "start" and "end" and try to recognize a * character class in expanded form. For example [0-9]. * On success, return the id the character class to be emitted. * On failure, return 0 (=FAIL) * Start points to the first char of the range, while end should point * to the closing brace. * Keep in mind that 'ignorecase' applies at execution time, thus [a-z] may * need to be interpreted as [a-zA-Z]. */ static int nfa_recognize_char_class(char_u *start, char_u *end, int extra_newl) { # define CLASS_not 0x80 # define CLASS_af 0x40 # define CLASS_AF 0x20 # define CLASS_az 0x10 # define CLASS_AZ 0x08 # define CLASS_o7 0x04 # define CLASS_o9 0x02 # define CLASS_underscore 0x01 int newl = FALSE; char_u *p; int config = 0; if (extra_newl == TRUE) newl = TRUE; if (*end != ']') return FAIL; p = start; if (*p == '^') { config |= CLASS_not; p++; } while (p < end) { if (p + 2 < end && *(p + 1) == '-') { switch (*p) { case '0': if (*(p + 2) == '9') { config |= CLASS_o9; break; } if (*(p + 2) == '7') { config |= CLASS_o7; break; } return FAIL; case 'a': if (*(p + 2) == 'z') { config |= CLASS_az; break; } if (*(p + 2) == 'f') { config |= CLASS_af; break; } return FAIL; case 'A': if (*(p + 2) == 'Z') { config |= CLASS_AZ; break; } if (*(p + 2) == 'F') { config |= CLASS_AF; break; } return FAIL; default: return FAIL; } p += 3; } else if (p + 1 < end && *p == '\\' && *(p + 1) == 'n') { newl = TRUE; p += 2; } else if (*p == '_') { config |= CLASS_underscore; p ++; } else if (*p == '\n') { newl = TRUE; p ++; } else return FAIL; } // while (p < end) if (p != end) return FAIL; if (newl == TRUE) extra_newl = NFA_ADD_NL; switch (config) { case CLASS_o9: return extra_newl + NFA_DIGIT; case CLASS_not | CLASS_o9: return extra_newl + NFA_NDIGIT; case CLASS_af | CLASS_AF | CLASS_o9: return extra_newl + NFA_HEX; case CLASS_not | CLASS_af | CLASS_AF | CLASS_o9: return extra_newl + NFA_NHEX; case CLASS_o7: return extra_newl + NFA_OCTAL; case CLASS_not | CLASS_o7: return extra_newl + NFA_NOCTAL; case CLASS_az | CLASS_AZ | CLASS_o9 | CLASS_underscore: return extra_newl + NFA_WORD; case CLASS_not | CLASS_az | CLASS_AZ | CLASS_o9 | CLASS_underscore: return extra_newl + NFA_NWORD; case CLASS_az | CLASS_AZ | CLASS_underscore: return extra_newl + NFA_HEAD; case CLASS_not | CLASS_az | CLASS_AZ | CLASS_underscore: return extra_newl + NFA_NHEAD; case CLASS_az | CLASS_AZ: return extra_newl + NFA_ALPHA; case CLASS_not | CLASS_az | CLASS_AZ: return extra_newl + NFA_NALPHA; case CLASS_az: return extra_newl + NFA_LOWER_IC; case CLASS_not | CLASS_az: return extra_newl + NFA_NLOWER_IC; case CLASS_AZ: return extra_newl + NFA_UPPER_IC; case CLASS_not | CLASS_AZ: return extra_newl + NFA_NUPPER_IC; } return FAIL; } /* * Produce the bytes for equivalence class "c". * Currently only handles latin1, latin9 and utf-8. * Emits bytes in postfix notation: 'a,b,NFA_OR,c,NFA_OR' is * equivalent to 'a OR b OR c' * * NOTE! When changing this function, also update reg_equi_class() */ static int nfa_emit_equi_class(int c) { #define EMIT2(c) EMIT(c); EMIT(NFA_CONCAT); if (enc_utf8 || STRCMP(p_enc, "latin1") == 0 || STRCMP(p_enc, "iso-8859-15") == 0) { #ifdef EBCDIC # define A_circumflex 0x62 # define A_diaeresis 0x63 # define A_grave 0x64 # define A_acute 0x65 # define A_virguilla 0x66 # define A_ring 0x67 # define C_cedilla 0x68 # define E_acute 0x71 # define E_circumflex 0x72 # define E_diaeresis 0x73 # define E_grave 0x74 # define I_acute 0x75 # define I_circumflex 0x76 # define I_diaeresis 0x77 # define I_grave 0x78 # define N_virguilla 0x69 # define O_circumflex 0xeb # define O_diaeresis 0xec # define O_grave 0xed # define O_acute 0xee # define O_virguilla 0xef # define O_slash 0x80 # define U_circumflex 0xfb # define U_diaeresis 0xfc # define U_grave 0xfd # define U_acute 0xfe # define Y_acute 0xba # define a_grave 0x42 # define a_acute 0x43 # define a_circumflex 0x44 # define a_virguilla 0x45 # define a_diaeresis 0x46 # define a_ring 0x47 # define c_cedilla 0x48 # define e_grave 0x51 # define e_acute 0x52 # define e_circumflex 0x53 # define e_diaeresis 0x54 # define i_grave 0x55 # define i_acute 0x56 # define i_circumflex 0x57 # define i_diaeresis 0x58 # define n_virguilla 0x49 # define o_grave 0xcb # define o_acute 0xcc # define o_circumflex 0xcd # define o_virguilla 0xce # define o_diaeresis 0xcf # define o_slash 0x70 # define u_grave 0xdb # define u_acute 0xdc # define u_circumflex 0xdd # define u_diaeresis 0xde # define y_acute 0x8d # define y_diaeresis 0xdf #else # define A_grave 0xc0 # define A_acute 0xc1 # define A_circumflex 0xc2 # define A_virguilla 0xc3 # define A_diaeresis 0xc4 # define A_ring 0xc5 # define C_cedilla 0xc7 # define E_grave 0xc8 # define E_acute 0xc9 # define E_circumflex 0xca # define E_diaeresis 0xcb # define I_grave 0xcc # define I_acute 0xcd # define I_circumflex 0xce # define I_diaeresis 0xcf # define N_virguilla 0xd1 # define O_grave 0xd2 # define O_acute 0xd3 # define O_circumflex 0xd4 # define O_virguilla 0xd5 # define O_diaeresis 0xd6 # define O_slash 0xd8 # define U_grave 0xd9 # define U_acute 0xda # define U_circumflex 0xdb # define U_diaeresis 0xdc # define Y_acute 0xdd # define a_grave 0xe0 # define a_acute 0xe1 # define a_circumflex 0xe2 # define a_virguilla 0xe3 # define a_diaeresis 0xe4 # define a_ring 0xe5 # define c_cedilla 0xe7 # define e_grave 0xe8 # define e_acute 0xe9 # define e_circumflex 0xea # define e_diaeresis 0xeb # define i_grave 0xec # define i_acute 0xed # define i_circumflex 0xee # define i_diaeresis 0xef # define n_virguilla 0xf1 # define o_grave 0xf2 # define o_acute 0xf3 # define o_circumflex 0xf4 # define o_virguilla 0xf5 # define o_diaeresis 0xf6 # define o_slash 0xf8 # define u_grave 0xf9 # define u_acute 0xfa # define u_circumflex 0xfb # define u_diaeresis 0xfc # define y_acute 0xfd # define y_diaeresis 0xff #endif switch (c) { case 'A': case A_grave: case A_acute: case A_circumflex: case A_virguilla: case A_diaeresis: case A_ring: case 0x100: case 0x102: case 0x104: case 0x1cd: case 0x1de: case 0x1e0: case 0x1fa: case 0x200: case 0x202: case 0x226: case 0x23a: case 0x1e00: case 0x1ea0: case 0x1ea2: case 0x1ea4: case 0x1ea6: case 0x1ea8: case 0x1eaa: case 0x1eac: case 0x1eae: case 0x1eb0: case 0x1eb2: case 0x1eb4: case 0x1eb6: EMIT2('A') EMIT2(A_grave) EMIT2(A_acute) EMIT2(A_circumflex) EMIT2(A_virguilla) EMIT2(A_diaeresis) EMIT2(A_ring) EMIT2(0x100) EMIT2(0x102) EMIT2(0x104) EMIT2(0x1cd) EMIT2(0x1de) EMIT2(0x1e0) EMIT2(0x1fa) EMIT2(0x200) EMIT2(0x202) EMIT2(0x226) EMIT2(0x23a) EMIT2(0x1e00) EMIT2(0x1ea0) EMIT2(0x1ea2) EMIT2(0x1ea4) EMIT2(0x1ea6) EMIT2(0x1ea8) EMIT2(0x1eaa) EMIT2(0x1eac) EMIT2(0x1eae) EMIT2(0x1eb0) EMIT2(0x1eb2) EMIT2(0x1eb6) EMIT2(0x1eb4) return OK; case 'B': case 0x181: case 0x243: case 0x1e02: case 0x1e04: case 0x1e06: EMIT2('B') EMIT2(0x181) EMIT2(0x243) EMIT2(0x1e02) EMIT2(0x1e04) EMIT2(0x1e06) return OK; case 'C': case C_cedilla: case 0x106: case 0x108: case 0x10a: case 0x10c: case 0x187: case 0x23b: case 0x1e08: case 0xa792: EMIT2('C') EMIT2(C_cedilla) EMIT2(0x106) EMIT2(0x108) EMIT2(0x10a) EMIT2(0x10c) EMIT2(0x187) EMIT2(0x23b) EMIT2(0x1e08) EMIT2(0xa792) return OK; case 'D': case 0x10e: case 0x110: case 0x18a: case 0x1e0a: case 0x1e0c: case 0x1e0e: case 0x1e10: case 0x1e12: EMIT2('D') EMIT2(0x10e) EMIT2(0x110) EMIT2(0x18a) EMIT2(0x1e0a) EMIT2(0x1e0c) EMIT2(0x1e0e) EMIT2(0x1e10) EMIT2(0x1e12) return OK; case 'E': case E_grave: case E_acute: case E_circumflex: case E_diaeresis: case 0x112: case 0x114: case 0x116: case 0x118: case 0x11a: case 0x204: case 0x206: case 0x228: case 0x246: case 0x1e14: case 0x1e16: case 0x1e18: case 0x1e1a: case 0x1e1c: case 0x1eb8: case 0x1eba: case 0x1ebc: case 0x1ebe: case 0x1ec0: case 0x1ec2: case 0x1ec4: case 0x1ec6: EMIT2('E') EMIT2(E_grave) EMIT2(E_acute) EMIT2(E_circumflex) EMIT2(E_diaeresis) EMIT2(0x112) EMIT2(0x114) EMIT2(0x116) EMIT2(0x118) EMIT2(0x11a) EMIT2(0x204) EMIT2(0x206) EMIT2(0x228) EMIT2(0x246) EMIT2(0x1e14) EMIT2(0x1e16) EMIT2(0x1e18) EMIT2(0x1e1a) EMIT2(0x1e1c) EMIT2(0x1eb8) EMIT2(0x1eba) EMIT2(0x1ebc) EMIT2(0x1ebe) EMIT2(0x1ec0) EMIT2(0x1ec2) EMIT2(0x1ec4) EMIT2(0x1ec6) return OK; case 'F': case 0x191: case 0x1e1e: case 0xa798: EMIT2('F') EMIT2(0x191) EMIT2(0x1e1e) EMIT2(0xa798) return OK; case 'G': case 0x11c: case 0x11e: case 0x120: case 0x122: case 0x193: case 0x1e4: case 0x1e6: case 0x1f4: case 0x1e20: case 0xa7a0: EMIT2('G') EMIT2(0x11c) EMIT2(0x11e) EMIT2(0x120) EMIT2(0x122) EMIT2(0x193) EMIT2(0x1e4) EMIT2(0x1e6) EMIT2(0x1f4) EMIT2(0x1e20) EMIT2(0xa7a0) return OK; case 'H': case 0x124: case 0x126: case 0x21e: case 0x1e22: case 0x1e24: case 0x1e26: case 0x1e28: case 0x1e2a: case 0x2c67: EMIT2('H') EMIT2(0x124) EMIT2(0x126) EMIT2(0x21e) EMIT2(0x1e22) EMIT2(0x1e24) EMIT2(0x1e26) EMIT2(0x1e28) EMIT2(0x1e2a) EMIT2(0x2c67) return OK; case 'I': case I_grave: case I_acute: case I_circumflex: case I_diaeresis: case 0x128: case 0x12a: case 0x12c: case 0x12e: case 0x130: case 0x197: case 0x1cf: case 0x208: case 0x20a: case 0x1e2c: case 0x1e2e: case 0x1ec8: case 0x1eca: EMIT2('I') EMIT2(I_grave) EMIT2(I_acute) EMIT2(I_circumflex) EMIT2(I_diaeresis) EMIT2(0x128) EMIT2(0x12a) EMIT2(0x12c) EMIT2(0x12e) EMIT2(0x130) EMIT2(0x197) EMIT2(0x1cf) EMIT2(0x208) EMIT2(0x20a) EMIT2(0x1e2c) EMIT2(0x1e2e) EMIT2(0x1ec8) EMIT2(0x1eca) return OK; case 'J': case 0x134: case 0x248: EMIT2('J') EMIT2(0x134) EMIT2(0x248) return OK; case 'K': case 0x136: case 0x198: case 0x1e8: case 0x1e30: case 0x1e32: case 0x1e34: case 0x2c69: case 0xa740: EMIT2('K') EMIT2(0x136) EMIT2(0x198) EMIT2(0x1e8) EMIT2(0x1e30) EMIT2(0x1e32) EMIT2(0x1e34) EMIT2(0x2c69) EMIT2(0xa740) return OK; case 'L': case 0x139: case 0x13b: case 0x13d: case 0x13f: case 0x141: case 0x23d: case 0x1e36: case 0x1e38: case 0x1e3a: case 0x1e3c: case 0x2c60: EMIT2('L') EMIT2(0x139) EMIT2(0x13b) EMIT2(0x13d) EMIT2(0x13f) EMIT2(0x141) EMIT2(0x23d) EMIT2(0x1e36) EMIT2(0x1e38) EMIT2(0x1e3a) EMIT2(0x1e3c) EMIT2(0x2c60) return OK; case 'M': case 0x1e3e: case 0x1e40: case 0x1e42: EMIT2('M') EMIT2(0x1e3e) EMIT2(0x1e40) EMIT2(0x1e42) return OK; case 'N': case N_virguilla: case 0x143: case 0x145: case 0x147: case 0x1f8: case 0x1e44: case 0x1e46: case 0x1e48: case 0x1e4a: case 0xa7a4: EMIT2('N') EMIT2(N_virguilla) EMIT2(0x143) EMIT2(0x145) EMIT2(0x147) EMIT2(0x1f8) EMIT2(0x1e44) EMIT2(0x1e46) EMIT2(0x1e48) EMIT2(0x1e4a) EMIT2(0xa7a4) return OK; case 'O': case O_grave: case O_acute: case O_circumflex: case O_virguilla: case O_diaeresis: case O_slash: case 0x14c: case 0x14e: case 0x150: case 0x19f: case 0x1a0: case 0x1d1: case 0x1ea: case 0x1ec: case 0x1fe: case 0x20c: case 0x20e: case 0x22a: case 0x22c: case 0x22e: case 0x230: case 0x1e4c: case 0x1e4e: case 0x1e50: case 0x1e52: case 0x1ecc: case 0x1ece: case 0x1ed0: case 0x1ed2: case 0x1ed4: case 0x1ed6: case 0x1ed8: case 0x1eda: case 0x1edc: case 0x1ede: case 0x1ee0: case 0x1ee2: EMIT2('O') EMIT2(O_grave) EMIT2(O_acute) EMIT2(O_circumflex) EMIT2(O_virguilla) EMIT2(O_diaeresis) EMIT2(O_slash) EMIT2(0x14c) EMIT2(0x14e) EMIT2(0x150) EMIT2(0x19f) EMIT2(0x1a0) EMIT2(0x1d1) EMIT2(0x1ea) EMIT2(0x1ec) EMIT2(0x1fe) EMIT2(0x20c) EMIT2(0x20e) EMIT2(0x22a) EMIT2(0x22c) EMIT2(0x22e) EMIT2(0x230) EMIT2(0x1e4c) EMIT2(0x1e4e) EMIT2(0x1e50) EMIT2(0x1e52) EMIT2(0x1ecc) EMIT2(0x1ece) EMIT2(0x1ed0) EMIT2(0x1ed2) EMIT2(0x1ed4) EMIT2(0x1ed6) EMIT2(0x1ed8) EMIT2(0x1eda) EMIT2(0x1edc) EMIT2(0x1ede) EMIT2(0x1ee0) EMIT2(0x1ee2) return OK; case 'P': case 0x1a4: case 0x1e54: case 0x1e56: case 0x2c63: EMIT2('P') EMIT2(0x1a4) EMIT2(0x1e54) EMIT2(0x1e56) EMIT2(0x2c63) return OK; case 'Q': case 0x24a: EMIT2('Q') EMIT2(0x24a) return OK; case 'R': case 0x154: case 0x156: case 0x158: case 0x210: case 0x212: case 0x24c: case 0x1e58: case 0x1e5a: case 0x1e5c: case 0x1e5e: case 0x2c64: case 0xa7a6: EMIT2('R') EMIT2(0x154) EMIT2(0x156) EMIT2(0x158) EMIT2(0x210) EMIT2(0x212) EMIT2(0x24c) EMIT2(0x1e58) EMIT2(0x1e5a) EMIT2(0x1e5c) EMIT2(0x1e5e) EMIT2(0x2c64) EMIT2(0xa7a6) return OK; case 'S': case 0x15a: case 0x15c: case 0x15e: case 0x160: case 0x218: case 0x1e60: case 0x1e62: case 0x1e64: case 0x1e66: case 0x1e68: case 0x2c7e: case 0xa7a8: EMIT2('S') EMIT2(0x15a) EMIT2(0x15c) EMIT2(0x15e) EMIT2(0x160) EMIT2(0x218) EMIT2(0x1e60) EMIT2(0x1e62) EMIT2(0x1e64) EMIT2(0x1e66) EMIT2(0x1e68) EMIT2(0x2c7e) EMIT2(0xa7a8) return OK; case 'T': case 0x162: case 0x164: case 0x166: case 0x1ac: case 0x1ae: case 0x21a: case 0x23e: case 0x1e6a: case 0x1e6c: case 0x1e6e: case 0x1e70: EMIT2('T') EMIT2(0x162) EMIT2(0x164) EMIT2(0x166) EMIT2(0x1ac) EMIT2(0x1ae) EMIT2(0x23e) EMIT2(0x21a) EMIT2(0x1e6a) EMIT2(0x1e6c) EMIT2(0x1e6e) EMIT2(0x1e70) return OK; case 'U': case U_grave: case U_acute: case U_diaeresis: case U_circumflex: case 0x168: case 0x16a: case 0x16c: case 0x16e: case 0x170: case 0x172: case 0x1af: case 0x1d3: case 0x1d5: case 0x1d7: case 0x1d9: case 0x1db: case 0x214: case 0x216: case 0x244: case 0x1e72: case 0x1e74: case 0x1e76: case 0x1e78: case 0x1e7a: case 0x1ee4: case 0x1ee6: case 0x1ee8: case 0x1eea: case 0x1eec: case 0x1eee: case 0x1ef0: EMIT2('U') EMIT2(U_grave) EMIT2(U_acute) EMIT2(U_diaeresis) EMIT2(U_circumflex) EMIT2(0x168) EMIT2(0x16a) EMIT2(0x16c) EMIT2(0x16e) EMIT2(0x170) EMIT2(0x172) EMIT2(0x1af) EMIT2(0x1d3) EMIT2(0x1d5) EMIT2(0x1d7) EMIT2(0x1d9) EMIT2(0x1db) EMIT2(0x214) EMIT2(0x216) EMIT2(0x244) EMIT2(0x1e72) EMIT2(0x1e74) EMIT2(0x1e76) EMIT2(0x1e78) EMIT2(0x1e7a) EMIT2(0x1ee4) EMIT2(0x1ee6) EMIT2(0x1ee8) EMIT2(0x1eea) EMIT2(0x1eec) EMIT2(0x1eee) EMIT2(0x1ef0) return OK; case 'V': case 0x1b2: case 0x1e7c: case 0x1e7e: EMIT2('V') EMIT2(0x1b2) EMIT2(0x1e7c) EMIT2(0x1e7e) return OK; case 'W': case 0x174: case 0x1e80: case 0x1e82: case 0x1e84: case 0x1e86: case 0x1e88: EMIT2('W') EMIT2(0x174) EMIT2(0x1e80) EMIT2(0x1e82) EMIT2(0x1e84) EMIT2(0x1e86) EMIT2(0x1e88) return OK; case 'X': case 0x1e8a: case 0x1e8c: EMIT2('X') EMIT2(0x1e8a) EMIT2(0x1e8c) return OK; case 'Y': case Y_acute: case 0x176: case 0x178: case 0x1b3: case 0x232: case 0x24e: case 0x1e8e: case 0x1ef2: case 0x1ef4: case 0x1ef6: case 0x1ef8: EMIT2('Y') EMIT2(Y_acute) EMIT2(0x176) EMIT2(0x178) EMIT2(0x1b3) EMIT2(0x232) EMIT2(0x24e) EMIT2(0x1e8e) EMIT2(0x1ef2) EMIT2(0x1ef4) EMIT2(0x1ef6) EMIT2(0x1ef8) return OK; case 'Z': case 0x179: case 0x17b: case 0x17d: case 0x1b5: case 0x1e90: case 0x1e92: case 0x1e94: case 0x2c6b: EMIT2('Z') EMIT2(0x179) EMIT2(0x17b) EMIT2(0x17d) EMIT2(0x1b5) EMIT2(0x1e90) EMIT2(0x1e92) EMIT2(0x1e94) EMIT2(0x2c6b) return OK; case 'a': case a_grave: case a_acute: case a_circumflex: case a_virguilla: case a_diaeresis: case a_ring: case 0x101: case 0x103: case 0x105: case 0x1ce: case 0x1df: case 0x1e1: case 0x1fb: case 0x201: case 0x203: case 0x227: case 0x1d8f: case 0x1e01: case 0x1e9a: case 0x1ea1: case 0x1ea3: case 0x1ea5: case 0x1ea7: case 0x1ea9: case 0x1eab: case 0x1ead: case 0x1eaf: case 0x1eb1: case 0x1eb3: case 0x1eb5: case 0x1eb7: case 0x2c65: EMIT2('a') EMIT2(a_grave) EMIT2(a_acute) EMIT2(a_circumflex) EMIT2(a_virguilla) EMIT2(a_diaeresis) EMIT2(a_ring) EMIT2(0x101) EMIT2(0x103) EMIT2(0x105) EMIT2(0x1ce) EMIT2(0x1df) EMIT2(0x1e1) EMIT2(0x1fb) EMIT2(0x201) EMIT2(0x203) EMIT2(0x227) EMIT2(0x1d8f) EMIT2(0x1e01) EMIT2(0x1e9a) EMIT2(0x1ea1) EMIT2(0x1ea3) EMIT2(0x1ea5) EMIT2(0x1ea7) EMIT2(0x1ea9) EMIT2(0x1eab) EMIT2(0x1ead) EMIT2(0x1eaf) EMIT2(0x1eb1) EMIT2(0x1eb3) EMIT2(0x1eb5) EMIT2(0x1eb7) EMIT2(0x2c65) return OK; case 'b': case 0x180: case 0x253: case 0x1d6c: case 0x1d80: case 0x1e03: case 0x1e05: case 0x1e07: EMIT2('b') EMIT2(0x180) EMIT2(0x253) EMIT2(0x1d6c) EMIT2(0x1d80) EMIT2(0x1e03) EMIT2(0x1e05) EMIT2(0x1e07) return OK; case 'c': case c_cedilla: case 0x107: case 0x109: case 0x10b: case 0x10d: case 0x188: case 0x23c: case 0x1e09: case 0xa793: case 0xa794: EMIT2('c') EMIT2(c_cedilla) EMIT2(0x107) EMIT2(0x109) EMIT2(0x10b) EMIT2(0x10d) EMIT2(0x188) EMIT2(0x23c) EMIT2(0x1e09) EMIT2(0xa793) EMIT2(0xa794) return OK; case 'd': case 0x10f: case 0x111: case 0x257: case 0x1d6d: case 0x1d81: case 0x1d91: case 0x1e0b: case 0x1e0d: case 0x1e0f: case 0x1e11: case 0x1e13: EMIT2('d') EMIT2(0x10f) EMIT2(0x111) EMIT2(0x257) EMIT2(0x1d6d) EMIT2(0x1d81) EMIT2(0x1d91) EMIT2(0x1e0b) EMIT2(0x1e0d) EMIT2(0x1e0f) EMIT2(0x1e11) EMIT2(0x1e13) return OK; case 'e': case e_grave: case e_acute: case e_circumflex: case e_diaeresis: case 0x113: case 0x115: case 0x117: case 0x119: case 0x11b: case 0x205: case 0x207: case 0x229: case 0x247: case 0x1d92: case 0x1e15: case 0x1e17: case 0x1e19: case 0x1e1b: case 0x1e1d: case 0x1eb9: case 0x1ebb: case 0x1ebd: case 0x1ebf: case 0x1ec1: case 0x1ec3: case 0x1ec5: case 0x1ec7: EMIT2('e') EMIT2(e_grave) EMIT2(e_acute) EMIT2(e_circumflex) EMIT2(e_diaeresis) EMIT2(0x113) EMIT2(0x115) EMIT2(0x117) EMIT2(0x119) EMIT2(0x11b) EMIT2(0x205) EMIT2(0x207) EMIT2(0x229) EMIT2(0x247) EMIT2(0x1d92) EMIT2(0x1e15) EMIT2(0x1e17) EMIT2(0x1e19) EMIT2(0x1e1b) EMIT2(0x1e1d) EMIT2(0x1eb9) EMIT2(0x1ebb) EMIT2(0x1ebd) EMIT2(0x1ebf) EMIT2(0x1ec1) EMIT2(0x1ec3) EMIT2(0x1ec5) EMIT2(0x1ec7) return OK; case 'f': case 0x192: case 0x1d6e: case 0x1d82: case 0x1e1f: case 0xa799: EMIT2('f') EMIT2(0x192) EMIT2(0x1d6e) EMIT2(0x1d82) EMIT2(0x1e1f) EMIT2(0xa799) return OK; case 'g': case 0x11d: case 0x11f: case 0x121: case 0x123: case 0x1e5: case 0x1e7: case 0x1f5: case 0x260: case 0x1d83: case 0x1e21: case 0xa7a1: EMIT2('g') EMIT2(0x11d) EMIT2(0x11f) EMIT2(0x121) EMIT2(0x123) EMIT2(0x1e5) EMIT2(0x1e7) EMIT2(0x1f5) EMIT2(0x260) EMIT2(0x1d83) EMIT2(0x1e21) EMIT2(0xa7a1) return OK; case 'h': case 0x125: case 0x127: case 0x21f: case 0x1e23: case 0x1e25: case 0x1e27: case 0x1e29: case 0x1e2b: case 0x1e96: case 0x2c68: case 0xa795: EMIT2('h') EMIT2(0x125) EMIT2(0x127) EMIT2(0x21f) EMIT2(0x1e23) EMIT2(0x1e25) EMIT2(0x1e27) EMIT2(0x1e29) EMIT2(0x1e2b) EMIT2(0x1e96) EMIT2(0x2c68) EMIT2(0xa795) return OK; case 'i': case i_grave: case i_acute: case i_circumflex: case i_diaeresis: case 0x129: case 0x12b: case 0x12d: case 0x12f: case 0x1d0: case 0x209: case 0x20b: case 0x268: case 0x1d96: case 0x1e2d: case 0x1e2f: case 0x1ec9: case 0x1ecb: EMIT2('i') EMIT2(i_grave) EMIT2(i_acute) EMIT2(i_circumflex) EMIT2(i_diaeresis) EMIT2(0x129) EMIT2(0x12b) EMIT2(0x12d) EMIT2(0x12f) EMIT2(0x1d0) EMIT2(0x209) EMIT2(0x20b) EMIT2(0x268) EMIT2(0x1d96) EMIT2(0x1e2d) EMIT2(0x1e2f) EMIT2(0x1ec9) EMIT2(0x1ecb) EMIT2(0x1ecb) return OK; case 'j': case 0x135: case 0x1f0: case 0x249: EMIT2('j') EMIT2(0x135) EMIT2(0x1f0) EMIT2(0x249) return OK; case 'k': case 0x137: case 0x199: case 0x1e9: case 0x1d84: case 0x1e31: case 0x1e33: case 0x1e35: case 0x2c6a: case 0xa741: EMIT2('k') EMIT2(0x137) EMIT2(0x199) EMIT2(0x1e9) EMIT2(0x1d84) EMIT2(0x1e31) EMIT2(0x1e33) EMIT2(0x1e35) EMIT2(0x2c6a) EMIT2(0xa741) return OK; case 'l': case 0x13a: case 0x13c: case 0x13e: case 0x140: case 0x142: case 0x19a: case 0x1e37: case 0x1e39: case 0x1e3b: case 0x1e3d: case 0x2c61: EMIT2('l') EMIT2(0x13a) EMIT2(0x13c) EMIT2(0x13e) EMIT2(0x140) EMIT2(0x142) EMIT2(0x19a) EMIT2(0x1e37) EMIT2(0x1e39) EMIT2(0x1e3b) EMIT2(0x1e3d) EMIT2(0x2c61) return OK; case 'm': case 0x1d6f: case 0x1e3f: case 0x1e41: case 0x1e43: EMIT2('m') EMIT2(0x1d6f) EMIT2(0x1e3f) EMIT2(0x1e41) EMIT2(0x1e43) return OK; case 'n': case n_virguilla: case 0x144: case 0x146: case 0x148: case 0x149: case 0x1f9: case 0x1d70: case 0x1d87: case 0x1e45: case 0x1e47: case 0x1e49: case 0x1e4b: case 0xa7a5: EMIT2('n') EMIT2(n_virguilla) EMIT2(0x144) EMIT2(0x146) EMIT2(0x148) EMIT2(0x149) EMIT2(0x1f9) EMIT2(0x1d70) EMIT2(0x1d87) EMIT2(0x1e45) EMIT2(0x1e47) EMIT2(0x1e49) EMIT2(0x1e4b) EMIT2(0xa7a5) return OK; case 'o': case o_grave: case o_acute: case o_circumflex: case o_virguilla: case o_diaeresis: case o_slash: case 0x14d: case 0x14f: case 0x151: case 0x1a1: case 0x1d2: case 0x1eb: case 0x1ed: case 0x1ff: case 0x20d: case 0x20f: case 0x22b: case 0x22d: case 0x22f: case 0x231: case 0x275: case 0x1e4d: case 0x1e4f: case 0x1e51: case 0x1e53: case 0x1ecd: case 0x1ecf: case 0x1ed1: case 0x1ed3: case 0x1ed5: case 0x1ed7: case 0x1ed9: case 0x1edb: case 0x1edd: case 0x1edf: case 0x1ee1: case 0x1ee3: EMIT2('o') EMIT2(o_grave) EMIT2(o_acute) EMIT2(o_circumflex) EMIT2(o_virguilla) EMIT2(o_diaeresis) EMIT2(o_slash) EMIT2(0x14d) EMIT2(0x14f) EMIT2(0x151) EMIT2(0x1a1) EMIT2(0x1d2) EMIT2(0x1eb) EMIT2(0x1ed) EMIT2(0x1ff) EMIT2(0x20d) EMIT2(0x20f) EMIT2(0x22b) EMIT2(0x22d) EMIT2(0x22f) EMIT2(0x231) EMIT2(0x275) EMIT2(0x1e4d) EMIT2(0x1e4f) EMIT2(0x1e51) EMIT2(0x1e53) EMIT2(0x1ecd) EMIT2(0x1ecf) EMIT2(0x1ed1) EMIT2(0x1ed3) EMIT2(0x1ed5) EMIT2(0x1ed7) EMIT2(0x1ed9) EMIT2(0x1edb) EMIT2(0x1edd) EMIT2(0x1edf) EMIT2(0x1ee1) EMIT2(0x1ee3) return OK; case 'p': case 0x1a5: case 0x1d71: case 0x1d7d: case 0x1d88: case 0x1e55: case 0x1e57: EMIT2('p') EMIT2(0x1a5) EMIT2(0x1d71) EMIT2(0x1d7d) EMIT2(0x1d88) EMIT2(0x1e55) EMIT2(0x1e57) return OK; case 'q': case 0x24b: case 0x2a0: EMIT2('q') EMIT2(0x24b) EMIT2(0x2a0) return OK; case 'r': case 0x155: case 0x157: case 0x159: case 0x211: case 0x213: case 0x24d: case 0x27d: case 0x1d72: case 0x1d73: case 0x1d89: case 0x1e59: case 0x1e5b: case 0x1e5d: case 0x1e5f: case 0xa7a7: EMIT2('r') EMIT2(0x155) EMIT2(0x157) EMIT2(0x159) EMIT2(0x211) EMIT2(0x213) EMIT2(0x24d) EMIT2(0x27d) EMIT2(0x1d72) EMIT2(0x1d73) EMIT2(0x1d89) EMIT2(0x1e59) EMIT2(0x1e5b) EMIT2(0x1e5d) EMIT2(0x1e5f) EMIT2(0xa7a7) return OK; case 's': case 0x15b: case 0x15d: case 0x15f: case 0x161: case 0x219: case 0x23f: case 0x1d74: case 0x1d8a: case 0x1e61: case 0x1e63: case 0x1e65: case 0x1e67: case 0x1e69: case 0xa7a9: EMIT2('s') EMIT2(0x15b) EMIT2(0x15d) EMIT2(0x15f) EMIT2(0x161) EMIT2(0x219) EMIT2(0x23f) EMIT2(0x1d74) EMIT2(0x1d8a) EMIT2(0x1e61) EMIT2(0x1e63) EMIT2(0x1e65) EMIT2(0x1e67) EMIT2(0x1e69) EMIT2(0xa7a9) return OK; case 't': case 0x163: case 0x165: case 0x167: case 0x1ab: case 0x1ad: case 0x21b: case 0x288: case 0x1d75: case 0x1e6b: case 0x1e6d: case 0x1e6f: case 0x1e71: case 0x1e97: case 0x2c66: EMIT2('t') EMIT2(0x163) EMIT2(0x165) EMIT2(0x167) EMIT2(0x1ab) EMIT2(0x1ad) EMIT2(0x21b) EMIT2(0x288) EMIT2(0x1d75) EMIT2(0x1e6b) EMIT2(0x1e6d) EMIT2(0x1e6f) EMIT2(0x1e71) EMIT2(0x1e97) EMIT2(0x2c66) return OK; case 'u': case u_grave: case u_acute: case u_circumflex: case u_diaeresis: case 0x169: case 0x16b: case 0x16d: case 0x16f: case 0x171: case 0x173: case 0x1b0: case 0x1d4: case 0x1d6: case 0x1d8: case 0x1da: case 0x1dc: case 0x215: case 0x217: case 0x289: case 0x1d7e: case 0x1d99: case 0x1e73: case 0x1e75: case 0x1e77: case 0x1e79: case 0x1e7b: case 0x1ee5: case 0x1ee7: case 0x1ee9: case 0x1eeb: case 0x1eed: case 0x1eef: case 0x1ef1: EMIT2('u') EMIT2(u_grave) EMIT2(u_acute) EMIT2(u_circumflex) EMIT2(u_diaeresis) EMIT2(0x169) EMIT2(0x16b) EMIT2(0x16d) EMIT2(0x16f) EMIT2(0x171) EMIT2(0x173) EMIT2(0x1d6) EMIT2(0x1d8) EMIT2(0x215) EMIT2(0x217) EMIT2(0x1b0) EMIT2(0x1d4) EMIT2(0x1da) EMIT2(0x1dc) EMIT2(0x289) EMIT2(0x1e73) EMIT2(0x1d7e) EMIT2(0x1d99) EMIT2(0x1e75) EMIT2(0x1e77) EMIT2(0x1e79) EMIT2(0x1e7b) EMIT2(0x1ee5) EMIT2(0x1ee7) EMIT2(0x1ee9) EMIT2(0x1eeb) EMIT2(0x1eed) EMIT2(0x1eef) EMIT2(0x1ef1) return OK; case 'v': case 0x28b: case 0x1d8c: case 0x1e7d: case 0x1e7f: EMIT2('v') EMIT2(0x28b) EMIT2(0x1d8c) EMIT2(0x1e7d) EMIT2(0x1e7f) return OK; case 'w': case 0x175: case 0x1e81: case 0x1e83: case 0x1e85: case 0x1e87: case 0x1e89: case 0x1e98: EMIT2('w') EMIT2(0x175) EMIT2(0x1e81) EMIT2(0x1e83) EMIT2(0x1e85) EMIT2(0x1e87) EMIT2(0x1e89) EMIT2(0x1e98) return OK; case 'x': case 0x1e8b: case 0x1e8d: EMIT2('x') EMIT2(0x1e8b) EMIT2(0x1e8d) return OK; case 'y': case y_acute: case y_diaeresis: case 0x177: case 0x1b4: case 0x233: case 0x24f: case 0x1e8f: case 0x1e99: case 0x1ef3: case 0x1ef5: case 0x1ef7: case 0x1ef9: EMIT2('y') EMIT2(y_acute) EMIT2(y_diaeresis) EMIT2(0x177) EMIT2(0x1b4) EMIT2(0x233) EMIT2(0x24f) EMIT2(0x1e8f) EMIT2(0x1e99) EMIT2(0x1ef3) EMIT2(0x1ef5) EMIT2(0x1ef7) EMIT2(0x1ef9) return OK; case 'z': case 0x17a: case 0x17c: case 0x17e: case 0x1b6: case 0x1d76: case 0x1d8e: case 0x1e91: case 0x1e93: case 0x1e95: case 0x2c6c: EMIT2('z') EMIT2(0x17a) EMIT2(0x17c) EMIT2(0x17e) EMIT2(0x1b6) EMIT2(0x1d76) EMIT2(0x1d8e) EMIT2(0x1e91) EMIT2(0x1e93) EMIT2(0x1e95) EMIT2(0x2c6c) return OK; // default: character itself } } EMIT2(c); return OK; #undef EMIT2 #undef EMIT2 } /* * Code to parse regular expression. * * We try to reuse parsing functions in regexp.c to * minimize surprise and keep the syntax consistent. */ /* * Parse the lowest level. * * An atom can be one of a long list of items. Many atoms match one character * in the text. It is often an ordinary character or a character class. * Braces can be used to make a pattern into an atom. The "\z(\)" construct * is only for syntax highlighting. * * atom ::= ordinary-atom * or \( pattern \) * or \%( pattern \) * or \z( pattern \) */ static int nfa_regatom(void) { int c; int charclass; int equiclass; int collclass; int got_coll_char; char_u *p; char_u *endp; char_u *old_regparse = regparse; int extra = 0; int emit_range; int negated; int result; int startc = -1; int endc = -1; int oldstartc = -1; int save_prev_at_start = prev_at_start; c = getchr(); switch (c) { case NUL: EMSG_RET_FAIL(_(e_nul_found)); case Magic('^'): EMIT(NFA_BOL); break; case Magic('$'): EMIT(NFA_EOL); #if defined(FEAT_SYN_HL) || defined(PROTO) had_eol = TRUE; #endif break; case Magic('<'): EMIT(NFA_BOW); break; case Magic('>'): EMIT(NFA_EOW); break; case Magic('_'): c = no_Magic(getchr()); if (c == NUL) EMSG_RET_FAIL(_(e_nul_found)); if (c == '^') // "\_^" is start-of-line { EMIT(NFA_BOL); break; } if (c == '$') // "\_$" is end-of-line { EMIT(NFA_EOL); #if defined(FEAT_SYN_HL) || defined(PROTO) had_eol = TRUE; #endif break; } extra = NFA_ADD_NL; // "\_[" is collection plus newline if (c == '[') goto collection; // "\_x" is character class plus newline // FALLTHROUGH /* * Character classes. */ case Magic('.'): case Magic('i'): case Magic('I'): case Magic('k'): case Magic('K'): case Magic('f'): case Magic('F'): case Magic('p'): case Magic('P'): case Magic('s'): case Magic('S'): case Magic('d'): case Magic('D'): case Magic('x'): case Magic('X'): case Magic('o'): case Magic('O'): case Magic('w'): case Magic('W'): case Magic('h'): case Magic('H'): case Magic('a'): case Magic('A'): case Magic('l'): case Magic('L'): case Magic('u'): case Magic('U'): p = vim_strchr(classchars, no_Magic(c)); if (p == NULL) { if (extra == NFA_ADD_NL) { semsg(_(e_ill_char_class), c); rc_did_emsg = TRUE; return FAIL; } siemsg("INTERNAL: Unknown character class char: %d", c); return FAIL; } // When '.' is followed by a composing char ignore the dot, so that // the composing char is matched here. if (enc_utf8 && c == Magic('.') && utf_iscomposing(peekchr())) { old_regparse = regparse; c = getchr(); goto nfa_do_multibyte; } EMIT(nfa_classcodes[p - classchars]); if (extra == NFA_ADD_NL) { EMIT(NFA_NEWL); EMIT(NFA_OR); regflags |= RF_HASNL; } break; case Magic('n'): if (reg_string) // In a string "\n" matches a newline character. EMIT(NL); else { // In buffer text "\n" matches the end of a line. EMIT(NFA_NEWL); regflags |= RF_HASNL; } break; case Magic('('): if (nfa_reg(REG_PAREN) == FAIL) return FAIL; // cascaded error break; case Magic('|'): case Magic('&'): case Magic(')'): semsg(_(e_misplaced), no_Magic(c)); return FAIL; case Magic('='): case Magic('?'): case Magic('+'): case Magic('@'): case Magic('*'): case Magic('{'): // these should follow an atom, not form an atom semsg(_(e_misplaced), no_Magic(c)); return FAIL; case Magic('~'): { char_u *lp; // Previous substitute pattern. // Generated as "\%(pattern\)". if (reg_prev_sub == NULL) { emsg(_(e_nopresub)); return FAIL; } for (lp = reg_prev_sub; *lp != NUL; MB_CPTR_ADV(lp)) { EMIT(PTR2CHAR(lp)); if (lp != reg_prev_sub) EMIT(NFA_CONCAT); } EMIT(NFA_NOPEN); break; } case Magic('1'): case Magic('2'): case Magic('3'): case Magic('4'): case Magic('5'): case Magic('6'): case Magic('7'): case Magic('8'): case Magic('9'): { int refnum = no_Magic(c) - '1'; if (!seen_endbrace(refnum + 1)) return FAIL; EMIT(NFA_BACKREF1 + refnum); rex.nfa_has_backref = TRUE; } break; case Magic('z'): c = no_Magic(getchr()); switch (c) { case 's': EMIT(NFA_ZSTART); if (re_mult_next("\\zs") == FAIL) return FAIL; break; case 'e': EMIT(NFA_ZEND); rex.nfa_has_zend = TRUE; if (re_mult_next("\\ze") == FAIL) return FAIL; break; #ifdef FEAT_SYN_HL case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': // \z1...\z9 if ((reg_do_extmatch & REX_USE) == 0) EMSG_RET_FAIL(_(e_z1_not_allowed)); EMIT(NFA_ZREF1 + (no_Magic(c) - '1')); // No need to set rex.nfa_has_backref, the sub-matches don't // change when \z1 .. \z9 matches or not. re_has_z = REX_USE; break; case '(': // \z( if ((reg_do_extmatch & REX_SET) == 0) EMSG_RET_FAIL(_(e_z_not_allowed)); if (nfa_reg(REG_ZPAREN) == FAIL) return FAIL; // cascaded error re_has_z = REX_SET; break; #endif default: semsg(_("E867: (NFA) Unknown operator '\\z%c'"), no_Magic(c)); return FAIL; } break; case Magic('%'): c = no_Magic(getchr()); switch (c) { // () without a back reference case '(': if (nfa_reg(REG_NPAREN) == FAIL) return FAIL; EMIT(NFA_NOPEN); break; case 'd': // %d123 decimal case 'o': // %o123 octal case 'x': // %xab hex 2 case 'u': // %uabcd hex 4 case 'U': // %U1234abcd hex 8 { long nr; switch (c) { case 'd': nr = getdecchrs(); break; case 'o': nr = getoctchrs(); break; case 'x': nr = gethexchrs(2); break; case 'u': nr = gethexchrs(4); break; case 'U': nr = gethexchrs(8); break; default: nr = -1; break; } if (nr < 0 || nr > INT_MAX) EMSG2_RET_FAIL( _("E678: Invalid character after %s%%[dxouU]"), reg_magic == MAGIC_ALL); // A NUL is stored in the text as NL // TODO: what if a composing character follows? EMIT(nr == 0 ? 0x0a : nr); } break; // Catch \%^ and \%$ regardless of where they appear in the // pattern -- regardless of whether or not it makes sense. case '^': EMIT(NFA_BOF); break; case '$': EMIT(NFA_EOF); break; case '#': EMIT(NFA_CURSOR); break; case 'V': EMIT(NFA_VISUAL); break; case 'C': EMIT(NFA_ANY_COMPOSING); break; case '[': { int n; // \%[abc] for (n = 0; (c = peekchr()) != ']'; ++n) { if (c == NUL) EMSG2_RET_FAIL(_(e_missing_sb), reg_magic == MAGIC_ALL); // recursive call! if (nfa_regatom() == FAIL) return FAIL; } getchr(); // get the ] if (n == 0) EMSG2_RET_FAIL(_(e_empty_sb), reg_magic == MAGIC_ALL); EMIT(NFA_OPT_CHARS); EMIT(n); // Emit as "\%(\%[abc]\)" to be able to handle // "\%[abc]*" which would cause the empty string to be // matched an unlimited number of times. NFA_NOPEN is // added only once at a position, while NFA_SPLIT is // added multiple times. This is more efficient than // not allowing NFA_SPLIT multiple times, it is used // a lot. EMIT(NFA_NOPEN); break; } default: { long_u n = 0; int cmp = c; if (c == '<' || c == '>') c = getchr(); while (VIM_ISDIGIT(c)) { long_u tmp = n * 10 + (c - '0'); if (tmp < n) { // overflow. emsg(_(e_value_too_large)); return FAIL; } n = tmp; c = getchr(); } if (c == 'l' || c == 'c' || c == 'v') { long_u limit = INT_MAX; if (c == 'l') { // \%{n}l \%{n}<l \%{n}>l EMIT(cmp == '<' ? NFA_LNUM_LT : cmp == '>' ? NFA_LNUM_GT : NFA_LNUM); if (save_prev_at_start) at_start = TRUE; } else if (c == 'c') // \%{n}c \%{n}<c \%{n}>c EMIT(cmp == '<' ? NFA_COL_LT : cmp == '>' ? NFA_COL_GT : NFA_COL); else { // \%{n}v \%{n}<v \%{n}>v EMIT(cmp == '<' ? NFA_VCOL_LT : cmp == '>' ? NFA_VCOL_GT : NFA_VCOL); limit = INT_MAX / MB_MAXBYTES; } if (n >= limit) { emsg(_(e_value_too_large)); return FAIL; } EMIT((int)n); break; } else if (c == '\'' && n == 0) { // \%'m \%<'m \%>'m EMIT(cmp == '<' ? NFA_MARK_LT : cmp == '>' ? NFA_MARK_GT : NFA_MARK); EMIT(getchr()); break; } } semsg(_("E867: (NFA) Unknown operator '\\%%%c'"), no_Magic(c)); return FAIL; } break; case Magic('['): collection: /* * [abc] uses NFA_START_COLL - NFA_END_COLL * [^abc] uses NFA_START_NEG_COLL - NFA_END_NEG_COLL * Each character is produced as a regular state, using * NFA_CONCAT to bind them together. * Besides normal characters there can be: * - character classes NFA_CLASS_* * - ranges, two characters followed by NFA_RANGE. */ p = regparse; endp = skip_anyof(p); if (*endp == ']') { /* * Try to reverse engineer character classes. For example, * recognize that [0-9] stands for \d and [A-Za-z_] for \h, * and perform the necessary substitutions in the NFA. */ result = nfa_recognize_char_class(regparse, endp, extra == NFA_ADD_NL); if (result != FAIL) { if (result >= NFA_FIRST_NL && result <= NFA_LAST_NL) { EMIT(result - NFA_ADD_NL); EMIT(NFA_NEWL); EMIT(NFA_OR); } else EMIT(result); regparse = endp; MB_PTR_ADV(regparse); return OK; } /* * Failed to recognize a character class. Use the simple * version that turns [abc] into 'a' OR 'b' OR 'c' */ startc = endc = oldstartc = -1; negated = FALSE; if (*regparse == '^') // negated range { negated = TRUE; MB_PTR_ADV(regparse); EMIT(NFA_START_NEG_COLL); } else EMIT(NFA_START_COLL); if (*regparse == '-') { startc = '-'; EMIT(startc); EMIT(NFA_CONCAT); MB_PTR_ADV(regparse); } // Emit the OR branches for each character in the [] emit_range = FALSE; while (regparse < endp) { oldstartc = startc; startc = -1; got_coll_char = FALSE; if (*regparse == '[') { // Check for [: :], [= =], [. .] equiclass = collclass = 0; charclass = get_char_class(®parse); if (charclass == CLASS_NONE) { equiclass = get_equi_class(®parse); if (equiclass == 0) collclass = get_coll_element(®parse); } // Character class like [:alpha:] if (charclass != CLASS_NONE) { switch (charclass) { case CLASS_ALNUM: EMIT(NFA_CLASS_ALNUM); break; case CLASS_ALPHA: EMIT(NFA_CLASS_ALPHA); break; case CLASS_BLANK: EMIT(NFA_CLASS_BLANK); break; case CLASS_CNTRL: EMIT(NFA_CLASS_CNTRL); break; case CLASS_DIGIT: EMIT(NFA_CLASS_DIGIT); break; case CLASS_GRAPH: EMIT(NFA_CLASS_GRAPH); break; case CLASS_LOWER: wants_nfa = TRUE; EMIT(NFA_CLASS_LOWER); break; case CLASS_PRINT: EMIT(NFA_CLASS_PRINT); break; case CLASS_PUNCT: EMIT(NFA_CLASS_PUNCT); break; case CLASS_SPACE: EMIT(NFA_CLASS_SPACE); break; case CLASS_UPPER: wants_nfa = TRUE; EMIT(NFA_CLASS_UPPER); break; case CLASS_XDIGIT: EMIT(NFA_CLASS_XDIGIT); break; case CLASS_TAB: EMIT(NFA_CLASS_TAB); break; case CLASS_RETURN: EMIT(NFA_CLASS_RETURN); break; case CLASS_BACKSPACE: EMIT(NFA_CLASS_BACKSPACE); break; case CLASS_ESCAPE: EMIT(NFA_CLASS_ESCAPE); break; case CLASS_IDENT: EMIT(NFA_CLASS_IDENT); break; case CLASS_KEYWORD: EMIT(NFA_CLASS_KEYWORD); break; case CLASS_FNAME: EMIT(NFA_CLASS_FNAME); break; } EMIT(NFA_CONCAT); continue; } // Try equivalence class [=a=] and the like if (equiclass != 0) { result = nfa_emit_equi_class(equiclass); if (result == FAIL) { // should never happen EMSG_RET_FAIL(_("E868: Error building NFA with equivalence class!")); } continue; } // Try collating class like [. .] if (collclass != 0) { startc = collclass; // allow [.a.]-x as a range // Will emit the proper atom at the end of the // while loop. } } // Try a range like 'a-x' or '\t-z'. Also allows '-' as a // start character. if (*regparse == '-' && oldstartc != -1) { emit_range = TRUE; startc = oldstartc; MB_PTR_ADV(regparse); continue; // reading the end of the range } // Now handle simple and escaped characters. // Only "\]", "\^", "\]" and "\\" are special in Vi. Vim // accepts "\t", "\e", etc., but only when the 'l' flag in // 'cpoptions' is not included. // Posix doesn't recognize backslash at all. if (*regparse == '\\' && !reg_cpo_bsl && regparse + 1 <= endp && (vim_strchr(REGEXP_INRANGE, regparse[1]) != NULL || (!reg_cpo_lit && vim_strchr(REGEXP_ABBR, regparse[1]) != NULL) ) ) { MB_PTR_ADV(regparse); if (*regparse == 'n') startc = (reg_string || emit_range || regparse[1] == '-') ? NL : NFA_NEWL; else if (*regparse == 'd' || *regparse == 'o' || *regparse == 'x' || *regparse == 'u' || *regparse == 'U' ) { // TODO(RE) This needs more testing startc = coll_get_char(); got_coll_char = TRUE; MB_PTR_BACK(old_regparse, regparse); } else { // \r,\t,\e,\b startc = backslash_trans(*regparse); } } // Normal printable char if (startc == -1) startc = PTR2CHAR(regparse); // Previous char was '-', so this char is end of range. if (emit_range) { endc = startc; startc = oldstartc; if (startc > endc) EMSG_RET_FAIL(_(e_reverse_range)); if (endc > startc + 2) { // Emit a range instead of the sequence of // individual characters. if (startc == 0) // \x00 is translated to \x0a, start at \x01. EMIT(1); else --post_ptr; // remove NFA_CONCAT EMIT(endc); EMIT(NFA_RANGE); EMIT(NFA_CONCAT); } else if (has_mbyte && ((*mb_char2len)(startc) > 1 || (*mb_char2len)(endc) > 1)) { // Emit the characters in the range. // "startc" was already emitted, so skip it. // for (c = startc + 1; c <= endc; c++) { EMIT(c); EMIT(NFA_CONCAT); } } else { #ifdef EBCDIC int alpha_only = FALSE; // for alphabetical range skip the gaps // 'i'-'j', 'r'-'s', 'I'-'J' and 'R'-'S'. if (isalpha(startc) && isalpha(endc)) alpha_only = TRUE; #endif // Emit the range. "startc" was already emitted, so // skip it. for (c = startc + 1; c <= endc; c++) #ifdef EBCDIC if (!alpha_only || isalpha(startc)) #endif { EMIT(c); EMIT(NFA_CONCAT); } } emit_range = FALSE; startc = -1; } else { // This char (startc) is not part of a range. Just // emit it. // Normally, simply emit startc. But if we get char // code=0 from a collating char, then replace it with // 0x0a. // This is needed to completely mimic the behaviour of // the backtracking engine. if (startc == NFA_NEWL) { // Line break can't be matched as part of the // collection, add an OR below. But not for negated // range. if (!negated) extra = NFA_ADD_NL; } else { if (got_coll_char == TRUE && startc == 0) EMIT(0x0a); else EMIT(startc); EMIT(NFA_CONCAT); } } MB_PTR_ADV(regparse); } // while (p < endp) MB_PTR_BACK(old_regparse, regparse); if (*regparse == '-') // if last, '-' is just a char { EMIT('-'); EMIT(NFA_CONCAT); } // skip the trailing ] regparse = endp; MB_PTR_ADV(regparse); // Mark end of the collection. if (negated == TRUE) EMIT(NFA_END_NEG_COLL); else EMIT(NFA_END_COLL); // \_[] also matches \n but it's not negated if (extra == NFA_ADD_NL) { EMIT(reg_string ? NL : NFA_NEWL); EMIT(NFA_OR); } return OK; } // if exists closing ] if (reg_strict) EMSG_RET_FAIL(_(e_missingbracket)); // FALLTHROUGH default: { int plen; nfa_do_multibyte: // plen is length of current char with composing chars if (enc_utf8 && ((*mb_char2len)(c) != (plen = utfc_ptr2len(old_regparse)) || utf_iscomposing(c))) { int i = 0; // A base character plus composing characters, or just one // or more composing characters. // This requires creating a separate atom as if enclosing // the characters in (), where NFA_COMPOSING is the ( and // NFA_END_COMPOSING is the ). Note that right now we are // building the postfix form, not the NFA itself; // a composing char could be: a, b, c, NFA_COMPOSING // where 'b' and 'c' are chars with codes > 256. for (;;) { EMIT(c); if (i > 0) EMIT(NFA_CONCAT); if ((i += utf_char2len(c)) >= plen) break; c = utf_ptr2char(old_regparse + i); } EMIT(NFA_COMPOSING); regparse = old_regparse + plen; } else { c = no_Magic(c); EMIT(c); } return OK; } } return OK; } /* * Parse something followed by possible [*+=]. * * A piece is an atom, possibly followed by a multi, an indication of how many * times the atom can be matched. Example: "a*" matches any sequence of "a" * characters: "", "a", "aa", etc. * * piece ::= atom * or atom multi */ static int nfa_regpiece(void) { int i; int op; int ret; long minval, maxval; int greedy = TRUE; // Braces are prefixed with '-' ? parse_state_T old_state; parse_state_T new_state; long c2; int old_post_pos; int my_post_start; int quest; // Save the current parse state, so that we can use it if <atom>{m,n} is // next. save_parse_state(&old_state); // store current pos in the postfix form, for \{m,n} involving 0s my_post_start = (int)(post_ptr - post_start); ret = nfa_regatom(); if (ret == FAIL) return FAIL; // cascaded error op = peekchr(); if (re_multi_type(op) == NOT_MULTI) return OK; skipchr(); switch (op) { case Magic('*'): EMIT(NFA_STAR); break; case Magic('+'): /* * Trick: Normally, (a*)\+ would match the whole input "aaa". The * first and only submatch would be "aaa". But the backtracking * engine interprets the plus as "try matching one more time", and * a* matches a second time at the end of the input, the empty * string. * The submatch will be the empty string. * * In order to be consistent with the old engine, we replace * <atom>+ with <atom><atom>* */ restore_parse_state(&old_state); curchr = -1; if (nfa_regatom() == FAIL) return FAIL; EMIT(NFA_STAR); EMIT(NFA_CONCAT); skipchr(); // skip the \+ break; case Magic('@'): c2 = getdecchrs(); op = no_Magic(getchr()); i = 0; switch(op) { case '=': // \@= i = NFA_PREV_ATOM_NO_WIDTH; break; case '!': // \@! i = NFA_PREV_ATOM_NO_WIDTH_NEG; break; case '<': op = no_Magic(getchr()); if (op == '=') // \@<= i = NFA_PREV_ATOM_JUST_BEFORE; else if (op == '!') // \@<! i = NFA_PREV_ATOM_JUST_BEFORE_NEG; break; case '>': // \@> i = NFA_PREV_ATOM_LIKE_PATTERN; break; } if (i == 0) { semsg(_("E869: (NFA) Unknown operator '\\@%c'"), op); return FAIL; } EMIT(i); if (i == NFA_PREV_ATOM_JUST_BEFORE || i == NFA_PREV_ATOM_JUST_BEFORE_NEG) EMIT(c2); break; case Magic('?'): case Magic('='): EMIT(NFA_QUEST); break; case Magic('{'): // a{2,5} will expand to 'aaa?a?a?' // a{-1,3} will expand to 'aa??a??', where ?? is the nongreedy // version of '?' // \v(ab){2,3} will expand to '(ab)(ab)(ab)?', where all the // parenthesis have the same id greedy = TRUE; c2 = peekchr(); if (c2 == '-' || c2 == Magic('-')) { skipchr(); greedy = FALSE; } if (!read_limits(&minval, &maxval)) EMSG_RET_FAIL(_("E870: (NFA regexp) Error reading repetition limits")); // <atom>{0,inf}, <atom>{0,} and <atom>{} are equivalent to // <atom>* if (minval == 0 && maxval == MAX_LIMIT) { if (greedy) // { { (match the braces) // \{}, \{0,} EMIT(NFA_STAR); else // { { (match the braces) // \{-}, \{-0,} EMIT(NFA_STAR_NONGREEDY); break; } // Special case: x{0} or x{-0} if (maxval == 0) { // Ignore result of previous call to nfa_regatom() post_ptr = post_start + my_post_start; // NFA_EMPTY is 0-length and works everywhere EMIT(NFA_EMPTY); return OK; } // The engine is very inefficient (uses too many states) when the // maximum is much larger than the minimum and when the maximum is // large. However, when maxval is MAX_LIMIT, it is okay, as this // will emit NFA_STAR. // Bail out if we can use the other engine, but only, when the // pattern does not need the NFA engine like (e.g. [[:upper:]]\{2,\} // does not work with with characters > 8 bit with the BT engine) if ((nfa_re_flags & RE_AUTO) && (maxval > 500 || maxval > minval + 200) && (maxval != MAX_LIMIT && minval < 200) && !wants_nfa) return FAIL; // Ignore previous call to nfa_regatom() post_ptr = post_start + my_post_start; // Save parse state after the repeated atom and the \{} save_parse_state(&new_state); quest = (greedy == TRUE? NFA_QUEST : NFA_QUEST_NONGREEDY); for (i = 0; i < maxval; i++) { // Goto beginning of the repeated atom restore_parse_state(&old_state); old_post_pos = (int)(post_ptr - post_start); if (nfa_regatom() == FAIL) return FAIL; // after "minval" times, atoms are optional if (i + 1 > minval) { if (maxval == MAX_LIMIT) { if (greedy) EMIT(NFA_STAR); else EMIT(NFA_STAR_NONGREEDY); } else EMIT(quest); } if (old_post_pos != my_post_start) EMIT(NFA_CONCAT); if (i + 1 > minval && maxval == MAX_LIMIT) break; } // Go to just after the repeated atom and the \{} restore_parse_state(&new_state); curchr = -1; break; default: break; } // end switch if (re_multi_type(peekchr()) != NOT_MULTI) // Can't have a multi follow a multi. EMSG_RET_FAIL(_("E871: (NFA regexp) Can't have a multi follow a multi")); return OK; } /* * Parse one or more pieces, concatenated. It matches a match for the * first piece, followed by a match for the second piece, etc. Example: * "f[0-9]b", first matches "f", then a digit and then "b". * * concat ::= piece * or piece piece * or piece piece piece * etc. */ static int nfa_regconcat(void) { int cont = TRUE; int first = TRUE; while (cont) { switch (peekchr()) { case NUL: case Magic('|'): case Magic('&'): case Magic(')'): cont = FALSE; break; case Magic('Z'): regflags |= RF_ICOMBINE; skipchr_keepstart(); break; case Magic('c'): regflags |= RF_ICASE; skipchr_keepstart(); break; case Magic('C'): regflags |= RF_NOICASE; skipchr_keepstart(); break; case Magic('v'): reg_magic = MAGIC_ALL; skipchr_keepstart(); curchr = -1; break; case Magic('m'): reg_magic = MAGIC_ON; skipchr_keepstart(); curchr = -1; break; case Magic('M'): reg_magic = MAGIC_OFF; skipchr_keepstart(); curchr = -1; break; case Magic('V'): reg_magic = MAGIC_NONE; skipchr_keepstart(); curchr = -1; break; default: if (nfa_regpiece() == FAIL) return FAIL; if (first == FALSE) EMIT(NFA_CONCAT); else first = FALSE; break; } } return OK; } /* * Parse a branch, one or more concats, separated by "\&". It matches the * last concat, but only if all the preceding concats also match at the same * position. Examples: * "foobeep\&..." matches "foo" in "foobeep". * ".*Peter\&.*Bob" matches in a line containing both "Peter" and "Bob" * * branch ::= concat * or concat \& concat * or concat \& concat \& concat * etc. */ static int nfa_regbranch(void) { int old_post_pos; old_post_pos = (int)(post_ptr - post_start); // First branch, possibly the only one if (nfa_regconcat() == FAIL) return FAIL; // Try next concats while (peekchr() == Magic('&')) { skipchr(); // if concat is empty do emit a node if (old_post_pos == (int)(post_ptr - post_start)) EMIT(NFA_EMPTY); EMIT(NFA_NOPEN); EMIT(NFA_PREV_ATOM_NO_WIDTH); old_post_pos = (int)(post_ptr - post_start); if (nfa_regconcat() == FAIL) return FAIL; // if concat is empty do emit a node if (old_post_pos == (int)(post_ptr - post_start)) EMIT(NFA_EMPTY); EMIT(NFA_CONCAT); } // if a branch is empty, emit one node for it if (old_post_pos == (int)(post_ptr - post_start)) EMIT(NFA_EMPTY); return OK; } /* * Parse a pattern, one or more branches, separated by "\|". It matches * anything that matches one of the branches. Example: "foo\|beep" matches * "foo" and matches "beep". If more than one branch matches, the first one * is used. * * pattern ::= branch * or branch \| branch * or branch \| branch \| branch * etc. */ static int nfa_reg( int paren) // REG_NOPAREN, REG_PAREN, REG_NPAREN or REG_ZPAREN { int parno = 0; if (paren == REG_PAREN) { if (regnpar >= NSUBEXP) // Too many `(' EMSG_RET_FAIL(_("E872: (NFA regexp) Too many '('")); parno = regnpar++; } #ifdef FEAT_SYN_HL else if (paren == REG_ZPAREN) { // Make a ZOPEN node. if (regnzpar >= NSUBEXP) EMSG_RET_FAIL(_("E879: (NFA regexp) Too many \\z(")); parno = regnzpar++; } #endif if (nfa_regbranch() == FAIL) return FAIL; // cascaded error while (peekchr() == Magic('|')) { skipchr(); if (nfa_regbranch() == FAIL) return FAIL; // cascaded error EMIT(NFA_OR); } // Check for proper termination. if (paren != REG_NOPAREN && getchr() != Magic(')')) { if (paren == REG_NPAREN) EMSG2_RET_FAIL(_(e_unmatchedpp), reg_magic == MAGIC_ALL); else EMSG2_RET_FAIL(_(e_unmatchedp), reg_magic == MAGIC_ALL); } else if (paren == REG_NOPAREN && peekchr() != NUL) { if (peekchr() == Magic(')')) EMSG2_RET_FAIL(_(e_unmatchedpar), reg_magic == MAGIC_ALL); else EMSG_RET_FAIL(_("E873: (NFA regexp) proper termination error")); } /* * Here we set the flag allowing back references to this set of * parentheses. */ if (paren == REG_PAREN) { had_endbrace[parno] = TRUE; // have seen the close paren EMIT(NFA_MOPEN + parno); } #ifdef FEAT_SYN_HL else if (paren == REG_ZPAREN) EMIT(NFA_ZOPEN + parno); #endif return OK; } #ifdef DEBUG static char_u code[50]; static void nfa_set_code(int c) { int addnl = FALSE; if (c >= NFA_FIRST_NL && c <= NFA_LAST_NL) { addnl = TRUE; c -= NFA_ADD_NL; } STRCPY(code, ""); switch (c) { case NFA_MATCH: STRCPY(code, "NFA_MATCH "); break; case NFA_SPLIT: STRCPY(code, "NFA_SPLIT "); break; case NFA_CONCAT: STRCPY(code, "NFA_CONCAT "); break; case NFA_NEWL: STRCPY(code, "NFA_NEWL "); break; case NFA_ZSTART: STRCPY(code, "NFA_ZSTART"); break; case NFA_ZEND: STRCPY(code, "NFA_ZEND"); break; case NFA_BACKREF1: STRCPY(code, "NFA_BACKREF1"); break; case NFA_BACKREF2: STRCPY(code, "NFA_BACKREF2"); break; case NFA_BACKREF3: STRCPY(code, "NFA_BACKREF3"); break; case NFA_BACKREF4: STRCPY(code, "NFA_BACKREF4"); break; case NFA_BACKREF5: STRCPY(code, "NFA_BACKREF5"); break; case NFA_BACKREF6: STRCPY(code, "NFA_BACKREF6"); break; case NFA_BACKREF7: STRCPY(code, "NFA_BACKREF7"); break; case NFA_BACKREF8: STRCPY(code, "NFA_BACKREF8"); break; case NFA_BACKREF9: STRCPY(code, "NFA_BACKREF9"); break; #ifdef FEAT_SYN_HL case NFA_ZREF1: STRCPY(code, "NFA_ZREF1"); break; case NFA_ZREF2: STRCPY(code, "NFA_ZREF2"); break; case NFA_ZREF3: STRCPY(code, "NFA_ZREF3"); break; case NFA_ZREF4: STRCPY(code, "NFA_ZREF4"); break; case NFA_ZREF5: STRCPY(code, "NFA_ZREF5"); break; case NFA_ZREF6: STRCPY(code, "NFA_ZREF6"); break; case NFA_ZREF7: STRCPY(code, "NFA_ZREF7"); break; case NFA_ZREF8: STRCPY(code, "NFA_ZREF8"); break; case NFA_ZREF9: STRCPY(code, "NFA_ZREF9"); break; #endif case NFA_SKIP: STRCPY(code, "NFA_SKIP"); break; case NFA_PREV_ATOM_NO_WIDTH: STRCPY(code, "NFA_PREV_ATOM_NO_WIDTH"); break; case NFA_PREV_ATOM_NO_WIDTH_NEG: STRCPY(code, "NFA_PREV_ATOM_NO_WIDTH_NEG"); break; case NFA_PREV_ATOM_JUST_BEFORE: STRCPY(code, "NFA_PREV_ATOM_JUST_BEFORE"); break; case NFA_PREV_ATOM_JUST_BEFORE_NEG: STRCPY(code, "NFA_PREV_ATOM_JUST_BEFORE_NEG"); break; case NFA_PREV_ATOM_LIKE_PATTERN: STRCPY(code, "NFA_PREV_ATOM_LIKE_PATTERN"); break; case NFA_NOPEN: STRCPY(code, "NFA_NOPEN"); break; case NFA_NCLOSE: STRCPY(code, "NFA_NCLOSE"); break; case NFA_START_INVISIBLE: STRCPY(code, "NFA_START_INVISIBLE"); break; case NFA_START_INVISIBLE_FIRST: STRCPY(code, "NFA_START_INVISIBLE_FIRST"); break; case NFA_START_INVISIBLE_NEG: STRCPY(code, "NFA_START_INVISIBLE_NEG"); break; case NFA_START_INVISIBLE_NEG_FIRST: STRCPY(code, "NFA_START_INVISIBLE_NEG_FIRST"); break; case NFA_START_INVISIBLE_BEFORE: STRCPY(code, "NFA_START_INVISIBLE_BEFORE"); break; case NFA_START_INVISIBLE_BEFORE_FIRST: STRCPY(code, "NFA_START_INVISIBLE_BEFORE_FIRST"); break; case NFA_START_INVISIBLE_BEFORE_NEG: STRCPY(code, "NFA_START_INVISIBLE_BEFORE_NEG"); break; case NFA_START_INVISIBLE_BEFORE_NEG_FIRST: STRCPY(code, "NFA_START_INVISIBLE_BEFORE_NEG_FIRST"); break; case NFA_START_PATTERN: STRCPY(code, "NFA_START_PATTERN"); break; case NFA_END_INVISIBLE: STRCPY(code, "NFA_END_INVISIBLE"); break; case NFA_END_INVISIBLE_NEG: STRCPY(code, "NFA_END_INVISIBLE_NEG"); break; case NFA_END_PATTERN: STRCPY(code, "NFA_END_PATTERN"); break; case NFA_COMPOSING: STRCPY(code, "NFA_COMPOSING"); break; case NFA_END_COMPOSING: STRCPY(code, "NFA_END_COMPOSING"); break; case NFA_OPT_CHARS: STRCPY(code, "NFA_OPT_CHARS"); break; case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: STRCPY(code, "NFA_MOPEN(x)"); code[10] = c - NFA_MOPEN + '0'; break; case NFA_MCLOSE: case NFA_MCLOSE1: case NFA_MCLOSE2: case NFA_MCLOSE3: case NFA_MCLOSE4: case NFA_MCLOSE5: case NFA_MCLOSE6: case NFA_MCLOSE7: case NFA_MCLOSE8: case NFA_MCLOSE9: STRCPY(code, "NFA_MCLOSE(x)"); code[11] = c - NFA_MCLOSE + '0'; break; #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: STRCPY(code, "NFA_ZOPEN(x)"); code[10] = c - NFA_ZOPEN + '0'; break; case NFA_ZCLOSE: case NFA_ZCLOSE1: case NFA_ZCLOSE2: case NFA_ZCLOSE3: case NFA_ZCLOSE4: case NFA_ZCLOSE5: case NFA_ZCLOSE6: case NFA_ZCLOSE7: case NFA_ZCLOSE8: case NFA_ZCLOSE9: STRCPY(code, "NFA_ZCLOSE(x)"); code[11] = c - NFA_ZCLOSE + '0'; break; #endif case NFA_EOL: STRCPY(code, "NFA_EOL "); break; case NFA_BOL: STRCPY(code, "NFA_BOL "); break; case NFA_EOW: STRCPY(code, "NFA_EOW "); break; case NFA_BOW: STRCPY(code, "NFA_BOW "); break; case NFA_EOF: STRCPY(code, "NFA_EOF "); break; case NFA_BOF: STRCPY(code, "NFA_BOF "); break; case NFA_LNUM: STRCPY(code, "NFA_LNUM "); break; case NFA_LNUM_GT: STRCPY(code, "NFA_LNUM_GT "); break; case NFA_LNUM_LT: STRCPY(code, "NFA_LNUM_LT "); break; case NFA_COL: STRCPY(code, "NFA_COL "); break; case NFA_COL_GT: STRCPY(code, "NFA_COL_GT "); break; case NFA_COL_LT: STRCPY(code, "NFA_COL_LT "); break; case NFA_VCOL: STRCPY(code, "NFA_VCOL "); break; case NFA_VCOL_GT: STRCPY(code, "NFA_VCOL_GT "); break; case NFA_VCOL_LT: STRCPY(code, "NFA_VCOL_LT "); break; case NFA_MARK: STRCPY(code, "NFA_MARK "); break; case NFA_MARK_GT: STRCPY(code, "NFA_MARK_GT "); break; case NFA_MARK_LT: STRCPY(code, "NFA_MARK_LT "); break; case NFA_CURSOR: STRCPY(code, "NFA_CURSOR "); break; case NFA_VISUAL: STRCPY(code, "NFA_VISUAL "); break; case NFA_ANY_COMPOSING: STRCPY(code, "NFA_ANY_COMPOSING "); break; case NFA_STAR: STRCPY(code, "NFA_STAR "); break; case NFA_STAR_NONGREEDY: STRCPY(code, "NFA_STAR_NONGREEDY "); break; case NFA_QUEST: STRCPY(code, "NFA_QUEST"); break; case NFA_QUEST_NONGREEDY: STRCPY(code, "NFA_QUEST_NON_GREEDY"); break; case NFA_EMPTY: STRCPY(code, "NFA_EMPTY"); break; case NFA_OR: STRCPY(code, "NFA_OR"); break; case NFA_START_COLL: STRCPY(code, "NFA_START_COLL"); break; case NFA_END_COLL: STRCPY(code, "NFA_END_COLL"); break; case NFA_START_NEG_COLL: STRCPY(code, "NFA_START_NEG_COLL"); break; case NFA_END_NEG_COLL: STRCPY(code, "NFA_END_NEG_COLL"); break; case NFA_RANGE: STRCPY(code, "NFA_RANGE"); break; case NFA_RANGE_MIN: STRCPY(code, "NFA_RANGE_MIN"); break; case NFA_RANGE_MAX: STRCPY(code, "NFA_RANGE_MAX"); break; case NFA_CLASS_ALNUM: STRCPY(code, "NFA_CLASS_ALNUM"); break; case NFA_CLASS_ALPHA: STRCPY(code, "NFA_CLASS_ALPHA"); break; case NFA_CLASS_BLANK: STRCPY(code, "NFA_CLASS_BLANK"); break; case NFA_CLASS_CNTRL: STRCPY(code, "NFA_CLASS_CNTRL"); break; case NFA_CLASS_DIGIT: STRCPY(code, "NFA_CLASS_DIGIT"); break; case NFA_CLASS_GRAPH: STRCPY(code, "NFA_CLASS_GRAPH"); break; case NFA_CLASS_LOWER: STRCPY(code, "NFA_CLASS_LOWER"); break; case NFA_CLASS_PRINT: STRCPY(code, "NFA_CLASS_PRINT"); break; case NFA_CLASS_PUNCT: STRCPY(code, "NFA_CLASS_PUNCT"); break; case NFA_CLASS_SPACE: STRCPY(code, "NFA_CLASS_SPACE"); break; case NFA_CLASS_UPPER: STRCPY(code, "NFA_CLASS_UPPER"); break; case NFA_CLASS_XDIGIT: STRCPY(code, "NFA_CLASS_XDIGIT"); break; case NFA_CLASS_TAB: STRCPY(code, "NFA_CLASS_TAB"); break; case NFA_CLASS_RETURN: STRCPY(code, "NFA_CLASS_RETURN"); break; case NFA_CLASS_BACKSPACE: STRCPY(code, "NFA_CLASS_BACKSPACE"); break; case NFA_CLASS_ESCAPE: STRCPY(code, "NFA_CLASS_ESCAPE"); break; case NFA_CLASS_IDENT: STRCPY(code, "NFA_CLASS_IDENT"); break; case NFA_CLASS_KEYWORD: STRCPY(code, "NFA_CLASS_KEYWORD"); break; case NFA_CLASS_FNAME: STRCPY(code, "NFA_CLASS_FNAME"); break; case NFA_ANY: STRCPY(code, "NFA_ANY"); break; case NFA_IDENT: STRCPY(code, "NFA_IDENT"); break; case NFA_SIDENT:STRCPY(code, "NFA_SIDENT"); break; case NFA_KWORD: STRCPY(code, "NFA_KWORD"); break; case NFA_SKWORD:STRCPY(code, "NFA_SKWORD"); break; case NFA_FNAME: STRCPY(code, "NFA_FNAME"); break; case NFA_SFNAME:STRCPY(code, "NFA_SFNAME"); break; case NFA_PRINT: STRCPY(code, "NFA_PRINT"); break; case NFA_SPRINT:STRCPY(code, "NFA_SPRINT"); break; case NFA_WHITE: STRCPY(code, "NFA_WHITE"); break; case NFA_NWHITE:STRCPY(code, "NFA_NWHITE"); break; case NFA_DIGIT: STRCPY(code, "NFA_DIGIT"); break; case NFA_NDIGIT:STRCPY(code, "NFA_NDIGIT"); break; case NFA_HEX: STRCPY(code, "NFA_HEX"); break; case NFA_NHEX: STRCPY(code, "NFA_NHEX"); break; case NFA_OCTAL: STRCPY(code, "NFA_OCTAL"); break; case NFA_NOCTAL:STRCPY(code, "NFA_NOCTAL"); break; case NFA_WORD: STRCPY(code, "NFA_WORD"); break; case NFA_NWORD: STRCPY(code, "NFA_NWORD"); break; case NFA_HEAD: STRCPY(code, "NFA_HEAD"); break; case NFA_NHEAD: STRCPY(code, "NFA_NHEAD"); break; case NFA_ALPHA: STRCPY(code, "NFA_ALPHA"); break; case NFA_NALPHA:STRCPY(code, "NFA_NALPHA"); break; case NFA_LOWER: STRCPY(code, "NFA_LOWER"); break; case NFA_NLOWER:STRCPY(code, "NFA_NLOWER"); break; case NFA_UPPER: STRCPY(code, "NFA_UPPER"); break; case NFA_NUPPER:STRCPY(code, "NFA_NUPPER"); break; case NFA_LOWER_IC: STRCPY(code, "NFA_LOWER_IC"); break; case NFA_NLOWER_IC: STRCPY(code, "NFA_NLOWER_IC"); break; case NFA_UPPER_IC: STRCPY(code, "NFA_UPPER_IC"); break; case NFA_NUPPER_IC: STRCPY(code, "NFA_NUPPER_IC"); break; default: STRCPY(code, "CHAR(x)"); code[5] = c; } if (addnl == TRUE) STRCAT(code, " + NEWLINE "); } #ifdef ENABLE_LOG static FILE *log_fd; static char_u e_log_open_failed[] = N_("Could not open temporary log file for writing, displaying on stderr... "); /* * Print the postfix notation of the current regexp. */ static void nfa_postfix_dump(char_u *expr, int retval) { int *p; FILE *f; f = fopen(NFA_REGEXP_DUMP_LOG, "a"); if (f != NULL) { fprintf(f, "\n-------------------------\n"); if (retval == FAIL) fprintf(f, ">>> NFA engine failed... \n"); else if (retval == OK) fprintf(f, ">>> NFA engine succeeded !\n"); fprintf(f, "Regexp: \"%s\"\nPostfix notation (char): \"", expr); for (p = post_start; *p && p < post_ptr; p++) { nfa_set_code(*p); fprintf(f, "%s, ", code); } fprintf(f, "\"\nPostfix notation (int): "); for (p = post_start; *p && p < post_ptr; p++) fprintf(f, "%d ", *p); fprintf(f, "\n\n"); fclose(f); } } /* * Print the NFA starting with a root node "state". */ static void nfa_print_state(FILE *debugf, nfa_state_T *state) { garray_T indent; ga_init2(&indent, 1, 64); ga_append(&indent, '\0'); nfa_print_state2(debugf, state, &indent); ga_clear(&indent); } static void nfa_print_state2(FILE *debugf, nfa_state_T *state, garray_T *indent) { char_u *p; if (state == NULL) return; fprintf(debugf, "(%2d)", abs(state->id)); // Output indent p = (char_u *)indent->ga_data; if (indent->ga_len >= 3) { int last = indent->ga_len - 3; char_u save[2]; STRNCPY(save, &p[last], 2); STRNCPY(&p[last], "+-", 2); fprintf(debugf, " %s", p); STRNCPY(&p[last], save, 2); } else fprintf(debugf, " %s", p); nfa_set_code(state->c); fprintf(debugf, "%s (%d) (id=%d) val=%d\n", code, state->c, abs(state->id), state->val); if (state->id < 0) return; state->id = abs(state->id) * -1; // grow indent for state->out indent->ga_len -= 1; if (state->out1) ga_concat(indent, (char_u *)"| "); else ga_concat(indent, (char_u *)" "); ga_append(indent, '\0'); nfa_print_state2(debugf, state->out, indent); // replace last part of indent for state->out1 indent->ga_len -= 3; ga_concat(indent, (char_u *)" "); ga_append(indent, '\0'); nfa_print_state2(debugf, state->out1, indent); // shrink indent indent->ga_len -= 3; ga_append(indent, '\0'); } /* * Print the NFA state machine. */ static void nfa_dump(nfa_regprog_T *prog) { FILE *debugf = fopen(NFA_REGEXP_DUMP_LOG, "a"); if (debugf != NULL) { nfa_print_state(debugf, prog->start); if (prog->reganch) fprintf(debugf, "reganch: %d\n", prog->reganch); if (prog->regstart != NUL) fprintf(debugf, "regstart: %c (decimal: %d)\n", prog->regstart, prog->regstart); if (prog->match_text != NULL) fprintf(debugf, "match_text: \"%s\"\n", prog->match_text); fclose(debugf); } } #endif // ENABLE_LOG #endif // DEBUG /* * Parse r.e. @expr and convert it into postfix form. * Return the postfix string on success, NULL otherwise. */ static int * re2post(void) { if (nfa_reg(REG_NOPAREN) == FAIL) return NULL; EMIT(NFA_MOPEN); return post_start; } // NB. Some of the code below is inspired by Russ's. /* * Represents an NFA state plus zero or one or two arrows exiting. * if c == MATCH, no arrows out; matching state. * If c == SPLIT, unlabeled arrows to out and out1 (if != NULL). * If c < 256, labeled arrow with character c to out. */ static nfa_state_T *state_ptr; // points to nfa_prog->state /* * Allocate and initialize nfa_state_T. */ static nfa_state_T * alloc_state(int c, nfa_state_T *out, nfa_state_T *out1) { nfa_state_T *s; if (istate >= nstate) return NULL; s = &state_ptr[istate++]; s->c = c; s->out = out; s->out1 = out1; s->val = 0; s->id = istate; s->lastlist[0] = 0; s->lastlist[1] = 0; return s; } /* * A partially built NFA without the matching state filled in. * Frag_T.start points at the start state. * Frag_T.out is a list of places that need to be set to the * next state for this fragment. */ // Since the out pointers in the list are always // uninitialized, we use the pointers themselves // as storage for the Ptrlists. typedef union Ptrlist Ptrlist; union Ptrlist { Ptrlist *next; nfa_state_T *s; }; struct Frag { nfa_state_T *start; Ptrlist *out; }; typedef struct Frag Frag_T; /* * Initialize a Frag_T struct and return it. */ static Frag_T frag(nfa_state_T *start, Ptrlist *out) { Frag_T n; n.start = start; n.out = out; return n; } /* * Create singleton list containing just outp. */ static Ptrlist * list1( nfa_state_T **outp) { Ptrlist *l; l = (Ptrlist *)outp; l->next = NULL; return l; } /* * Patch the list of states at out to point to start. */ static void patch(Ptrlist *l, nfa_state_T *s) { Ptrlist *next; for (; l; l = next) { next = l->next; l->s = s; } } /* * Join the two lists l1 and l2, returning the combination. */ static Ptrlist * append(Ptrlist *l1, Ptrlist *l2) { Ptrlist *oldl1; oldl1 = l1; while (l1->next) l1 = l1->next; l1->next = l2; return oldl1; } /* * Stack used for transforming postfix form into NFA. */ static Frag_T empty; static void st_error(int *postfix UNUSED, int *end UNUSED, int *p UNUSED) { #ifdef NFA_REGEXP_ERROR_LOG FILE *df; int *p2; df = fopen(NFA_REGEXP_ERROR_LOG, "a"); if (df) { fprintf(df, "Error popping the stack!\n"); # ifdef DEBUG fprintf(df, "Current regexp is \"%s\"\n", nfa_regengine.expr); # endif fprintf(df, "Postfix form is: "); # ifdef DEBUG for (p2 = postfix; p2 < end; p2++) { nfa_set_code(*p2); fprintf(df, "%s, ", code); } nfa_set_code(*p); fprintf(df, "\nCurrent position is: "); for (p2 = postfix; p2 <= p; p2 ++) { nfa_set_code(*p2); fprintf(df, "%s, ", code); } # else for (p2 = postfix; p2 < end; p2++) fprintf(df, "%d, ", *p2); fprintf(df, "\nCurrent position is: "); for (p2 = postfix; p2 <= p; p2 ++) fprintf(df, "%d, ", *p2); # endif fprintf(df, "\n--------------------------\n"); fclose(df); } #endif emsg(_("E874: (NFA) Could not pop the stack!")); } /* * Push an item onto the stack. */ static void st_push(Frag_T s, Frag_T **p, Frag_T *stack_end) { Frag_T *stackp = *p; if (stackp >= stack_end) return; *stackp = s; *p = *p + 1; } /* * Pop an item from the stack. */ static Frag_T st_pop(Frag_T **p, Frag_T *stack) { Frag_T *stackp; *p = *p - 1; stackp = *p; if (stackp < stack) return empty; return **p; } /* * Estimate the maximum byte length of anything matching "state". * When unknown or unlimited return -1. */ static int nfa_max_width(nfa_state_T *startstate, int depth) { int l, r; nfa_state_T *state = startstate; int len = 0; // detect looping in a NFA_SPLIT if (depth > 4) return -1; while (state != NULL) { switch (state->c) { case NFA_END_INVISIBLE: case NFA_END_INVISIBLE_NEG: // the end, return what we have return len; case NFA_SPLIT: // two alternatives, use the maximum l = nfa_max_width(state->out, depth + 1); r = nfa_max_width(state->out1, depth + 1); if (l < 0 || r < 0) return -1; return len + (l > r ? l : r); case NFA_ANY: case NFA_START_COLL: case NFA_START_NEG_COLL: // matches some character, including composing chars if (enc_utf8) len += MB_MAXBYTES; else if (has_mbyte) len += 2; else ++len; if (state->c != NFA_ANY) { // skip over the characters state = state->out1->out; continue; } break; case NFA_DIGIT: case NFA_WHITE: case NFA_HEX: case NFA_OCTAL: // ascii ++len; break; case NFA_IDENT: case NFA_SIDENT: case NFA_KWORD: case NFA_SKWORD: case NFA_FNAME: case NFA_SFNAME: case NFA_PRINT: case NFA_SPRINT: case NFA_NWHITE: case NFA_NDIGIT: case NFA_NHEX: case NFA_NOCTAL: case NFA_WORD: case NFA_NWORD: case NFA_HEAD: case NFA_NHEAD: case NFA_ALPHA: case NFA_NALPHA: case NFA_LOWER: case NFA_NLOWER: case NFA_UPPER: case NFA_NUPPER: case NFA_LOWER_IC: case NFA_NLOWER_IC: case NFA_UPPER_IC: case NFA_NUPPER_IC: case NFA_ANY_COMPOSING: // possibly non-ascii if (has_mbyte) len += 3; else ++len; break; case NFA_START_INVISIBLE: case NFA_START_INVISIBLE_NEG: case NFA_START_INVISIBLE_BEFORE: case NFA_START_INVISIBLE_BEFORE_NEG: // zero-width, out1 points to the END state state = state->out1->out; continue; case NFA_BACKREF1: case NFA_BACKREF2: case NFA_BACKREF3: case NFA_BACKREF4: case NFA_BACKREF5: case NFA_BACKREF6: case NFA_BACKREF7: case NFA_BACKREF8: case NFA_BACKREF9: #ifdef FEAT_SYN_HL case NFA_ZREF1: case NFA_ZREF2: case NFA_ZREF3: case NFA_ZREF4: case NFA_ZREF5: case NFA_ZREF6: case NFA_ZREF7: case NFA_ZREF8: case NFA_ZREF9: #endif case NFA_NEWL: case NFA_SKIP: // unknown width return -1; case NFA_BOL: case NFA_EOL: case NFA_BOF: case NFA_EOF: case NFA_BOW: case NFA_EOW: case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: case NFA_ZCLOSE: case NFA_ZCLOSE1: case NFA_ZCLOSE2: case NFA_ZCLOSE3: case NFA_ZCLOSE4: case NFA_ZCLOSE5: case NFA_ZCLOSE6: case NFA_ZCLOSE7: case NFA_ZCLOSE8: case NFA_ZCLOSE9: #endif case NFA_MCLOSE: case NFA_MCLOSE1: case NFA_MCLOSE2: case NFA_MCLOSE3: case NFA_MCLOSE4: case NFA_MCLOSE5: case NFA_MCLOSE6: case NFA_MCLOSE7: case NFA_MCLOSE8: case NFA_MCLOSE9: case NFA_NOPEN: case NFA_NCLOSE: case NFA_LNUM_GT: case NFA_LNUM_LT: case NFA_COL_GT: case NFA_COL_LT: case NFA_VCOL_GT: case NFA_VCOL_LT: case NFA_MARK_GT: case NFA_MARK_LT: case NFA_VISUAL: case NFA_LNUM: case NFA_CURSOR: case NFA_COL: case NFA_VCOL: case NFA_MARK: case NFA_ZSTART: case NFA_ZEND: case NFA_OPT_CHARS: case NFA_EMPTY: case NFA_START_PATTERN: case NFA_END_PATTERN: case NFA_COMPOSING: case NFA_END_COMPOSING: // zero-width break; default: if (state->c < 0) // don't know what this is return -1; // normal character len += MB_CHAR2LEN(state->c); break; } // normal way to continue state = state->out; } // unrecognized, "cannot happen" return -1; } /* * Convert a postfix form into its equivalent NFA. * Return the NFA start state on success, NULL otherwise. */ static nfa_state_T * post2nfa(int *postfix, int *end, int nfa_calc_size) { int *p; int mopen; int mclose; Frag_T *stack = NULL; Frag_T *stackp = NULL; Frag_T *stack_end = NULL; Frag_T e1; Frag_T e2; Frag_T e; nfa_state_T *s; nfa_state_T *s1; nfa_state_T *matchstate; nfa_state_T *ret = NULL; if (postfix == NULL) return NULL; #define PUSH(s) st_push((s), &stackp, stack_end) #define POP() st_pop(&stackp, stack); \ if (stackp < stack) \ { \ st_error(postfix, end, p); \ vim_free(stack); \ return NULL; \ } if (nfa_calc_size == FALSE) { // Allocate space for the stack. Max states on the stack: "nstate". stack = ALLOC_MULT(Frag_T, nstate + 1); if (stack == NULL) return NULL; stackp = stack; stack_end = stack + (nstate + 1); } for (p = postfix; p < end; ++p) { switch (*p) { case NFA_CONCAT: // Concatenation. // Pay attention: this operator does not exist in the r.e. itself // (it is implicit, really). It is added when r.e. is translated // to postfix form in re2post(). if (nfa_calc_size == TRUE) { // nstate += 0; break; } e2 = POP(); e1 = POP(); patch(e1.out, e2.start); PUSH(frag(e1.start, e2.out)); break; case NFA_OR: // Alternation if (nfa_calc_size == TRUE) { nstate++; break; } e2 = POP(); e1 = POP(); s = alloc_state(NFA_SPLIT, e1.start, e2.start); if (s == NULL) goto theend; PUSH(frag(s, append(e1.out, e2.out))); break; case NFA_STAR: // Zero or more, prefer more if (nfa_calc_size == TRUE) { nstate++; break; } e = POP(); s = alloc_state(NFA_SPLIT, e.start, NULL); if (s == NULL) goto theend; patch(e.out, s); PUSH(frag(s, list1(&s->out1))); break; case NFA_STAR_NONGREEDY: // Zero or more, prefer zero if (nfa_calc_size == TRUE) { nstate++; break; } e = POP(); s = alloc_state(NFA_SPLIT, NULL, e.start); if (s == NULL) goto theend; patch(e.out, s); PUSH(frag(s, list1(&s->out))); break; case NFA_QUEST: // one or zero atoms=> greedy match if (nfa_calc_size == TRUE) { nstate++; break; } e = POP(); s = alloc_state(NFA_SPLIT, e.start, NULL); if (s == NULL) goto theend; PUSH(frag(s, append(e.out, list1(&s->out1)))); break; case NFA_QUEST_NONGREEDY: // zero or one atoms => non-greedy match if (nfa_calc_size == TRUE) { nstate++; break; } e = POP(); s = alloc_state(NFA_SPLIT, NULL, e.start); if (s == NULL) goto theend; PUSH(frag(s, append(e.out, list1(&s->out)))); break; case NFA_END_COLL: case NFA_END_NEG_COLL: // On the stack is the sequence starting with NFA_START_COLL or // NFA_START_NEG_COLL and all possible characters. Patch it to // add the output to the start. if (nfa_calc_size == TRUE) { nstate++; break; } e = POP(); s = alloc_state(NFA_END_COLL, NULL, NULL); if (s == NULL) goto theend; patch(e.out, s); e.start->out1 = s; PUSH(frag(e.start, list1(&s->out))); break; case NFA_RANGE: // Before this are two characters, the low and high end of a // range. Turn them into two states with MIN and MAX. if (nfa_calc_size == TRUE) { // nstate += 0; break; } e2 = POP(); e1 = POP(); e2.start->val = e2.start->c; e2.start->c = NFA_RANGE_MAX; e1.start->val = e1.start->c; e1.start->c = NFA_RANGE_MIN; patch(e1.out, e2.start); PUSH(frag(e1.start, e2.out)); break; case NFA_EMPTY: // 0-length, used in a repetition with max/min count of 0 if (nfa_calc_size == TRUE) { nstate++; break; } s = alloc_state(NFA_EMPTY, NULL, NULL); if (s == NULL) goto theend; PUSH(frag(s, list1(&s->out))); break; case NFA_OPT_CHARS: { int n; // \%[abc] implemented as: // NFA_SPLIT // +-CHAR(a) // | +-NFA_SPLIT // | +-CHAR(b) // | | +-NFA_SPLIT // | | +-CHAR(c) // | | | +-next // | | +- next // | +- next // +- next n = *++p; // get number of characters if (nfa_calc_size == TRUE) { nstate += n; break; } s = NULL; // avoid compiler warning e1.out = NULL; // stores list with out1's s1 = NULL; // previous NFA_SPLIT to connect to while (n-- > 0) { e = POP(); // get character s = alloc_state(NFA_SPLIT, e.start, NULL); if (s == NULL) goto theend; if (e1.out == NULL) e1 = e; patch(e.out, s1); append(e1.out, list1(&s->out1)); s1 = s; } PUSH(frag(s, e1.out)); break; } case NFA_PREV_ATOM_NO_WIDTH: case NFA_PREV_ATOM_NO_WIDTH_NEG: case NFA_PREV_ATOM_JUST_BEFORE: case NFA_PREV_ATOM_JUST_BEFORE_NEG: case NFA_PREV_ATOM_LIKE_PATTERN: { int before = (*p == NFA_PREV_ATOM_JUST_BEFORE || *p == NFA_PREV_ATOM_JUST_BEFORE_NEG); int pattern = (*p == NFA_PREV_ATOM_LIKE_PATTERN); int start_state; int end_state; int n = 0; nfa_state_T *zend; nfa_state_T *skip; switch (*p) { case NFA_PREV_ATOM_NO_WIDTH: start_state = NFA_START_INVISIBLE; end_state = NFA_END_INVISIBLE; break; case NFA_PREV_ATOM_NO_WIDTH_NEG: start_state = NFA_START_INVISIBLE_NEG; end_state = NFA_END_INVISIBLE_NEG; break; case NFA_PREV_ATOM_JUST_BEFORE: start_state = NFA_START_INVISIBLE_BEFORE; end_state = NFA_END_INVISIBLE; break; case NFA_PREV_ATOM_JUST_BEFORE_NEG: start_state = NFA_START_INVISIBLE_BEFORE_NEG; end_state = NFA_END_INVISIBLE_NEG; break; default: // NFA_PREV_ATOM_LIKE_PATTERN: start_state = NFA_START_PATTERN; end_state = NFA_END_PATTERN; break; } if (before) n = *++p; // get the count // The \@= operator: match the preceding atom with zero width. // The \@! operator: no match for the preceding atom. // The \@<= operator: match for the preceding atom. // The \@<! operator: no match for the preceding atom. // Surrounds the preceding atom with START_INVISIBLE and // END_INVISIBLE, similarly to MOPEN. if (nfa_calc_size == TRUE) { nstate += pattern ? 4 : 2; break; } e = POP(); s1 = alloc_state(end_state, NULL, NULL); if (s1 == NULL) goto theend; s = alloc_state(start_state, e.start, s1); if (s == NULL) goto theend; if (pattern) { // NFA_ZEND -> NFA_END_PATTERN -> NFA_SKIP -> what follows. skip = alloc_state(NFA_SKIP, NULL, NULL); if (skip == NULL) goto theend; zend = alloc_state(NFA_ZEND, s1, NULL); if (zend == NULL) goto theend; s1->out= skip; patch(e.out, zend); PUSH(frag(s, list1(&skip->out))); } else { patch(e.out, s1); PUSH(frag(s, list1(&s1->out))); if (before) { if (n <= 0) // See if we can guess the maximum width, it avoids a // lot of pointless tries. n = nfa_max_width(e.start, 0); s->val = n; // store the count } } break; } case NFA_COMPOSING: // char with composing char #if 0 // TODO if (regflags & RF_ICOMBINE) { // use the base character only } #endif // FALLTHROUGH case NFA_MOPEN: // \( \) Submatch case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: // \z( \) Submatch case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif case NFA_NOPEN: // \%( \) "Invisible Submatch" if (nfa_calc_size == TRUE) { nstate += 2; break; } mopen = *p; switch (*p) { case NFA_NOPEN: mclose = NFA_NCLOSE; break; #ifdef FEAT_SYN_HL case NFA_ZOPEN: mclose = NFA_ZCLOSE; break; case NFA_ZOPEN1: mclose = NFA_ZCLOSE1; break; case NFA_ZOPEN2: mclose = NFA_ZCLOSE2; break; case NFA_ZOPEN3: mclose = NFA_ZCLOSE3; break; case NFA_ZOPEN4: mclose = NFA_ZCLOSE4; break; case NFA_ZOPEN5: mclose = NFA_ZCLOSE5; break; case NFA_ZOPEN6: mclose = NFA_ZCLOSE6; break; case NFA_ZOPEN7: mclose = NFA_ZCLOSE7; break; case NFA_ZOPEN8: mclose = NFA_ZCLOSE8; break; case NFA_ZOPEN9: mclose = NFA_ZCLOSE9; break; #endif case NFA_COMPOSING: mclose = NFA_END_COMPOSING; break; default: // NFA_MOPEN, NFA_MOPEN1 .. NFA_MOPEN9 mclose = *p + NSUBEXP; break; } // Allow "NFA_MOPEN" as a valid postfix representation for // the empty regexp "". In this case, the NFA will be // NFA_MOPEN -> NFA_MCLOSE. Note that this also allows // empty groups of parenthesis, and empty mbyte chars if (stackp == stack) { s = alloc_state(mopen, NULL, NULL); if (s == NULL) goto theend; s1 = alloc_state(mclose, NULL, NULL); if (s1 == NULL) goto theend; patch(list1(&s->out), s1); PUSH(frag(s, list1(&s1->out))); break; } // At least one node was emitted before NFA_MOPEN, so // at least one node will be between NFA_MOPEN and NFA_MCLOSE e = POP(); s = alloc_state(mopen, e.start, NULL); // `(' if (s == NULL) goto theend; s1 = alloc_state(mclose, NULL, NULL); // `)' if (s1 == NULL) goto theend; patch(e.out, s1); if (mopen == NFA_COMPOSING) // COMPOSING->out1 = END_COMPOSING patch(list1(&s->out1), s1); PUSH(frag(s, list1(&s1->out))); break; case NFA_BACKREF1: case NFA_BACKREF2: case NFA_BACKREF3: case NFA_BACKREF4: case NFA_BACKREF5: case NFA_BACKREF6: case NFA_BACKREF7: case NFA_BACKREF8: case NFA_BACKREF9: #ifdef FEAT_SYN_HL case NFA_ZREF1: case NFA_ZREF2: case NFA_ZREF3: case NFA_ZREF4: case NFA_ZREF5: case NFA_ZREF6: case NFA_ZREF7: case NFA_ZREF8: case NFA_ZREF9: #endif if (nfa_calc_size == TRUE) { nstate += 2; break; } s = alloc_state(*p, NULL, NULL); if (s == NULL) goto theend; s1 = alloc_state(NFA_SKIP, NULL, NULL); if (s1 == NULL) goto theend; patch(list1(&s->out), s1); PUSH(frag(s, list1(&s1->out))); break; case NFA_LNUM: case NFA_LNUM_GT: case NFA_LNUM_LT: case NFA_VCOL: case NFA_VCOL_GT: case NFA_VCOL_LT: case NFA_COL: case NFA_COL_GT: case NFA_COL_LT: case NFA_MARK: case NFA_MARK_GT: case NFA_MARK_LT: { int n = *++p; // lnum, col or mark name if (nfa_calc_size == TRUE) { nstate += 1; break; } s = alloc_state(p[-1], NULL, NULL); if (s == NULL) goto theend; s->val = n; PUSH(frag(s, list1(&s->out))); break; } case NFA_ZSTART: case NFA_ZEND: default: // Operands if (nfa_calc_size == TRUE) { nstate++; break; } s = alloc_state(*p, NULL, NULL); if (s == NULL) goto theend; PUSH(frag(s, list1(&s->out))); break; } // switch(*p) } // for(p = postfix; *p; ++p) if (nfa_calc_size == TRUE) { nstate++; goto theend; // Return value when counting size is ignored anyway } e = POP(); if (stackp != stack) { vim_free(stack); EMSG_RET_NULL(_("E875: (NFA regexp) (While converting from postfix to NFA), too many states left on stack")); } if (istate >= nstate) { vim_free(stack); EMSG_RET_NULL(_("E876: (NFA regexp) Not enough space to store the whole NFA ")); } matchstate = &state_ptr[istate++]; // the match state matchstate->c = NFA_MATCH; matchstate->out = matchstate->out1 = NULL; matchstate->id = 0; patch(e.out, matchstate); ret = e.start; theend: vim_free(stack); return ret; #undef POP1 #undef PUSH1 #undef POP2 #undef PUSH2 #undef POP #undef PUSH } /* * After building the NFA program, inspect it to add optimization hints. */ static void nfa_postprocess(nfa_regprog_T *prog) { int i; int c; for (i = 0; i < prog->nstate; ++i) { c = prog->state[i].c; if (c == NFA_START_INVISIBLE || c == NFA_START_INVISIBLE_NEG || c == NFA_START_INVISIBLE_BEFORE || c == NFA_START_INVISIBLE_BEFORE_NEG) { int directly; // Do it directly when what follows is possibly the end of the // match. if (match_follows(prog->state[i].out1->out, 0)) directly = TRUE; else { int ch_invisible = failure_chance(prog->state[i].out, 0); int ch_follows = failure_chance(prog->state[i].out1->out, 0); // Postpone when the invisible match is expensive or has a // lower chance of failing. if (c == NFA_START_INVISIBLE_BEFORE || c == NFA_START_INVISIBLE_BEFORE_NEG) { // "before" matches are very expensive when // unbounded, always prefer what follows then, // unless what follows will always match. // Otherwise strongly prefer what follows. if (prog->state[i].val <= 0 && ch_follows > 0) directly = FALSE; else directly = ch_follows * 10 < ch_invisible; } else { // normal invisible, first do the one with the // highest failure chance directly = ch_follows < ch_invisible; } } if (directly) // switch to the _FIRST state ++prog->state[i].c; } } } ///////////////////////////////////////////////////////////////// // NFA execution code. ///////////////////////////////////////////////////////////////// typedef struct { int in_use; // number of subexpr with useful info // When REG_MULTI is TRUE list.multi is used, otherwise list.line. union { struct multipos { linenr_T start_lnum; linenr_T end_lnum; colnr_T start_col; colnr_T end_col; } multi[NSUBEXP]; struct linepos { char_u *start; char_u *end; } line[NSUBEXP]; } list; } regsub_T; typedef struct { regsub_T norm; // \( .. \) matches #ifdef FEAT_SYN_HL regsub_T synt; // \z( .. \) matches #endif } regsubs_T; // nfa_pim_T stores a Postponed Invisible Match. typedef struct nfa_pim_S nfa_pim_T; struct nfa_pim_S { int result; // NFA_PIM_*, see below nfa_state_T *state; // the invisible match start state regsubs_T subs; // submatch info, only party used union { lpos_T pos; char_u *ptr; } end; // where the match must end }; // Values for done in nfa_pim_T. #define NFA_PIM_UNUSED 0 // pim not used #define NFA_PIM_TODO 1 // pim not done yet #define NFA_PIM_MATCH 2 // pim executed, matches #define NFA_PIM_NOMATCH 3 // pim executed, no match // nfa_thread_T contains execution information of a NFA state typedef struct { nfa_state_T *state; int count; nfa_pim_T pim; // if pim.result != NFA_PIM_UNUSED: postponed // invisible match regsubs_T subs; // submatch info, only party used } nfa_thread_T; // nfa_list_T contains the alternative NFA execution states. typedef struct { nfa_thread_T *t; // allocated array of states int n; // nr of states currently in "t" int len; // max nr of states in "t" int id; // ID of the list int has_pim; // TRUE when any state has a PIM } nfa_list_T; #ifdef ENABLE_LOG static void log_subexpr(regsub_T *sub); static void log_subsexpr(regsubs_T *subs) { log_subexpr(&subs->norm); # ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) log_subexpr(&subs->synt); # endif } static void log_subexpr(regsub_T *sub) { int j; for (j = 0; j < sub->in_use; j++) if (REG_MULTI) fprintf(log_fd, "*** group %d, start: c=%d, l=%d, end: c=%d, l=%d\n", j, sub->list.multi[j].start_col, (int)sub->list.multi[j].start_lnum, sub->list.multi[j].end_col, (int)sub->list.multi[j].end_lnum); else { char *s = (char *)sub->list.line[j].start; char *e = (char *)sub->list.line[j].end; fprintf(log_fd, "*** group %d, start: \"%s\", end: \"%s\"\n", j, s == NULL ? "NULL" : s, e == NULL ? "NULL" : e); } } static char * pim_info(nfa_pim_T *pim) { static char buf[30]; if (pim == NULL || pim->result == NFA_PIM_UNUSED) buf[0] = NUL; else { sprintf(buf, " PIM col %d", REG_MULTI ? (int)pim->end.pos.col : (int)(pim->end.ptr - rex.input)); } return buf; } #endif // Used during execution: whether a match has been found. static int nfa_match; #ifdef FEAT_RELTIME static proftime_T *nfa_time_limit; static int *nfa_timed_out; static int nfa_time_count; #endif static void copy_sub(regsub_T *to, regsub_T *from); static int pim_equal(nfa_pim_T *one, nfa_pim_T *two); /* * Copy postponed invisible match info from "from" to "to". */ static void copy_pim(nfa_pim_T *to, nfa_pim_T *from) { to->result = from->result; to->state = from->state; copy_sub(&to->subs.norm, &from->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub(&to->subs.synt, &from->subs.synt); #endif to->end = from->end; } static void clear_sub(regsub_T *sub) { if (REG_MULTI) // Use 0xff to set lnum to -1 vim_memset(sub->list.multi, 0xff, sizeof(struct multipos) * rex.nfa_nsubexpr); else vim_memset(sub->list.line, 0, sizeof(struct linepos) * rex.nfa_nsubexpr); sub->in_use = 0; } /* * Copy the submatches from "from" to "to". */ static void copy_sub(regsub_T *to, regsub_T *from) { to->in_use = from->in_use; if (from->in_use > 0) { // Copy the match start and end positions. if (REG_MULTI) mch_memmove(&to->list.multi[0], &from->list.multi[0], sizeof(struct multipos) * from->in_use); else mch_memmove(&to->list.line[0], &from->list.line[0], sizeof(struct linepos) * from->in_use); } } /* * Like copy_sub() but exclude the main match. */ static void copy_sub_off(regsub_T *to, regsub_T *from) { if (to->in_use < from->in_use) to->in_use = from->in_use; if (from->in_use > 1) { // Copy the match start and end positions. if (REG_MULTI) mch_memmove(&to->list.multi[1], &from->list.multi[1], sizeof(struct multipos) * (from->in_use - 1)); else mch_memmove(&to->list.line[1], &from->list.line[1], sizeof(struct linepos) * (from->in_use - 1)); } } /* * Like copy_sub() but only do the end of the main match if \ze is present. */ static void copy_ze_off(regsub_T *to, regsub_T *from) { if (rex.nfa_has_zend) { if (REG_MULTI) { if (from->list.multi[0].end_lnum >= 0) { to->list.multi[0].end_lnum = from->list.multi[0].end_lnum; to->list.multi[0].end_col = from->list.multi[0].end_col; } } else { if (from->list.line[0].end != NULL) to->list.line[0].end = from->list.line[0].end; } } } /* * Return TRUE if "sub1" and "sub2" have the same start positions. * When using back-references also check the end position. */ static int sub_equal(regsub_T *sub1, regsub_T *sub2) { int i; int todo; linenr_T s1; linenr_T s2; char_u *sp1; char_u *sp2; todo = sub1->in_use > sub2->in_use ? sub1->in_use : sub2->in_use; if (REG_MULTI) { for (i = 0; i < todo; ++i) { if (i < sub1->in_use) s1 = sub1->list.multi[i].start_lnum; else s1 = -1; if (i < sub2->in_use) s2 = sub2->list.multi[i].start_lnum; else s2 = -1; if (s1 != s2) return FALSE; if (s1 != -1 && sub1->list.multi[i].start_col != sub2->list.multi[i].start_col) return FALSE; if (rex.nfa_has_backref) { if (i < sub1->in_use) s1 = sub1->list.multi[i].end_lnum; else s1 = -1; if (i < sub2->in_use) s2 = sub2->list.multi[i].end_lnum; else s2 = -1; if (s1 != s2) return FALSE; if (s1 != -1 && sub1->list.multi[i].end_col != sub2->list.multi[i].end_col) return FALSE; } } } else { for (i = 0; i < todo; ++i) { if (i < sub1->in_use) sp1 = sub1->list.line[i].start; else sp1 = NULL; if (i < sub2->in_use) sp2 = sub2->list.line[i].start; else sp2 = NULL; if (sp1 != sp2) return FALSE; if (rex.nfa_has_backref) { if (i < sub1->in_use) sp1 = sub1->list.line[i].end; else sp1 = NULL; if (i < sub2->in_use) sp2 = sub2->list.line[i].end; else sp2 = NULL; if (sp1 != sp2) return FALSE; } } } return TRUE; } #ifdef ENABLE_LOG static void report_state(char *action, regsub_T *sub, nfa_state_T *state, int lid, nfa_pim_T *pim) { int col; if (sub->in_use <= 0) col = -1; else if (REG_MULTI) col = sub->list.multi[0].start_col; else col = (int)(sub->list.line[0].start - rex.line); nfa_set_code(state->c); fprintf(log_fd, "> %s state %d to list %d. char %d: %s (start col %d)%s\n", action, abs(state->id), lid, state->c, code, col, pim_info(pim)); } #endif /* * Return TRUE if the same state is already in list "l" with the same * positions as "subs". */ static int has_state_with_pos( nfa_list_T *l, // runtime state list nfa_state_T *state, // state to update regsubs_T *subs, // pointers to subexpressions nfa_pim_T *pim) // postponed match or NULL { nfa_thread_T *thread; int i; for (i = 0; i < l->n; ++i) { thread = &l->t[i]; if (thread->state->id == state->id && sub_equal(&thread->subs.norm, &subs->norm) #ifdef FEAT_SYN_HL && (!rex.nfa_has_zsubexpr || sub_equal(&thread->subs.synt, &subs->synt)) #endif && pim_equal(&thread->pim, pim)) return TRUE; } return FALSE; } /* * Return TRUE if "one" and "two" are equal. That includes when both are not * set. */ static int pim_equal(nfa_pim_T *one, nfa_pim_T *two) { int one_unused = (one == NULL || one->result == NFA_PIM_UNUSED); int two_unused = (two == NULL || two->result == NFA_PIM_UNUSED); if (one_unused) // one is unused: equal when two is also unused return two_unused; if (two_unused) // one is used and two is not: not equal return FALSE; // compare the state id if (one->state->id != two->state->id) return FALSE; // compare the position if (REG_MULTI) return one->end.pos.lnum == two->end.pos.lnum && one->end.pos.col == two->end.pos.col; return one->end.ptr == two->end.ptr; } /* * Return TRUE if "state" leads to a NFA_MATCH without advancing the input. */ static int match_follows(nfa_state_T *startstate, int depth) { nfa_state_T *state = startstate; // avoid too much recursion if (depth > 10) return FALSE; while (state != NULL) { switch (state->c) { case NFA_MATCH: case NFA_MCLOSE: case NFA_END_INVISIBLE: case NFA_END_INVISIBLE_NEG: case NFA_END_PATTERN: return TRUE; case NFA_SPLIT: return match_follows(state->out, depth + 1) || match_follows(state->out1, depth + 1); case NFA_START_INVISIBLE: case NFA_START_INVISIBLE_FIRST: case NFA_START_INVISIBLE_BEFORE: case NFA_START_INVISIBLE_BEFORE_FIRST: case NFA_START_INVISIBLE_NEG: case NFA_START_INVISIBLE_NEG_FIRST: case NFA_START_INVISIBLE_BEFORE_NEG: case NFA_START_INVISIBLE_BEFORE_NEG_FIRST: case NFA_COMPOSING: // skip ahead to next state state = state->out1->out; continue; case NFA_ANY: case NFA_ANY_COMPOSING: case NFA_IDENT: case NFA_SIDENT: case NFA_KWORD: case NFA_SKWORD: case NFA_FNAME: case NFA_SFNAME: case NFA_PRINT: case NFA_SPRINT: case NFA_WHITE: case NFA_NWHITE: case NFA_DIGIT: case NFA_NDIGIT: case NFA_HEX: case NFA_NHEX: case NFA_OCTAL: case NFA_NOCTAL: case NFA_WORD: case NFA_NWORD: case NFA_HEAD: case NFA_NHEAD: case NFA_ALPHA: case NFA_NALPHA: case NFA_LOWER: case NFA_NLOWER: case NFA_UPPER: case NFA_NUPPER: case NFA_LOWER_IC: case NFA_NLOWER_IC: case NFA_UPPER_IC: case NFA_NUPPER_IC: case NFA_START_COLL: case NFA_START_NEG_COLL: case NFA_NEWL: // state will advance input return FALSE; default: if (state->c > 0) // state will advance input return FALSE; // Others: zero-width or possibly zero-width, might still find // a match at the same position, keep looking. break; } state = state->out; } return FALSE; } /* * Return TRUE if "state" is already in list "l". */ static int state_in_list( nfa_list_T *l, // runtime state list nfa_state_T *state, // state to update regsubs_T *subs) // pointers to subexpressions { if (state->lastlist[nfa_ll_index] == l->id) { if (!rex.nfa_has_backref || has_state_with_pos(l, state, subs, NULL)) return TRUE; } return FALSE; } // Offset used for "off" by addstate_here(). #define ADDSTATE_HERE_OFFSET 10 /* * Add "state" and possibly what follows to state list ".". * Returns "subs_arg", possibly copied into temp_subs. * Returns NULL when recursiveness is too deep. */ static regsubs_T * addstate( nfa_list_T *l, // runtime state list nfa_state_T *state, // state to update regsubs_T *subs_arg, // pointers to subexpressions nfa_pim_T *pim, // postponed look-behind match int off_arg) // byte offset, when -1 go to next line { int subidx; int off = off_arg; int add_here = FALSE; int listindex = 0; int k; int found = FALSE; nfa_thread_T *thread; struct multipos save_multipos; int save_in_use; char_u *save_ptr; int i; regsub_T *sub; regsubs_T *subs = subs_arg; static regsubs_T temp_subs; #ifdef ENABLE_LOG int did_print = FALSE; #endif static int depth = 0; // This function is called recursively. When the depth is too much we run // out of stack and crash, limit recursiveness here. if (++depth >= 5000 || subs == NULL) { --depth; return NULL; } if (off_arg <= -ADDSTATE_HERE_OFFSET) { add_here = TRUE; off = 0; listindex = -(off_arg + ADDSTATE_HERE_OFFSET); } switch (state->c) { case NFA_NCLOSE: case NFA_MCLOSE: case NFA_MCLOSE1: case NFA_MCLOSE2: case NFA_MCLOSE3: case NFA_MCLOSE4: case NFA_MCLOSE5: case NFA_MCLOSE6: case NFA_MCLOSE7: case NFA_MCLOSE8: case NFA_MCLOSE9: #ifdef FEAT_SYN_HL case NFA_ZCLOSE: case NFA_ZCLOSE1: case NFA_ZCLOSE2: case NFA_ZCLOSE3: case NFA_ZCLOSE4: case NFA_ZCLOSE5: case NFA_ZCLOSE6: case NFA_ZCLOSE7: case NFA_ZCLOSE8: case NFA_ZCLOSE9: #endif case NFA_MOPEN: case NFA_ZEND: case NFA_SPLIT: case NFA_EMPTY: // These nodes are not added themselves but their "out" and/or // "out1" may be added below. break; case NFA_BOL: case NFA_BOF: // "^" won't match past end-of-line, don't bother trying. // Except when at the end of the line, or when we are going to the // next line for a look-behind match. if (rex.input > rex.line && *rex.input != NUL && (nfa_endp == NULL || !REG_MULTI || rex.lnum == nfa_endp->se_u.pos.lnum)) goto skip_add; // FALLTHROUGH case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif case NFA_NOPEN: case NFA_ZSTART: // These nodes need to be added so that we can bail out when it // was added to this list before at the same position to avoid an // endless loop for "\(\)*" default: if (state->lastlist[nfa_ll_index] == l->id && state->c != NFA_SKIP) { // This state is already in the list, don't add it again, // unless it is an MOPEN that is used for a backreference or // when there is a PIM. For NFA_MATCH check the position, // lower position is preferred. if (!rex.nfa_has_backref && pim == NULL && !l->has_pim && state->c != NFA_MATCH) { // When called from addstate_here() do insert before // existing states. if (add_here) { for (k = 0; k < l->n && k < listindex; ++k) if (l->t[k].state->id == state->id) { found = TRUE; break; } } if (!add_here || found) { skip_add: #ifdef ENABLE_LOG nfa_set_code(state->c); fprintf(log_fd, "> Not adding state %d to list %d. char %d: %s pim: %s has_pim: %d found: %d\n", abs(state->id), l->id, state->c, code, pim == NULL ? "NULL" : "yes", l->has_pim, found); #endif --depth; return subs; } } // Do not add the state again when it exists with the same // positions. if (has_state_with_pos(l, state, subs, pim)) goto skip_add; } // When there are backreferences or PIMs the number of states may // be (a lot) bigger than anticipated. if (l->n == l->len) { int newlen = l->len * 3 / 2 + 50; size_t newsize = newlen * sizeof(nfa_thread_T); nfa_thread_T *newt; if ((long)(newsize >> 10) >= p_mmp) { emsg(_(e_maxmempat)); --depth; return NULL; } if (subs != &temp_subs) { // "subs" may point into the current array, need to make a // copy before it becomes invalid. copy_sub(&temp_subs.norm, &subs->norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub(&temp_subs.synt, &subs->synt); #endif subs = &temp_subs; } newt = vim_realloc(l->t, newsize); if (newt == NULL) { // out of memory --depth; return NULL; } l->t = newt; l->len = newlen; } // add the state to the list state->lastlist[nfa_ll_index] = l->id; thread = &l->t[l->n++]; thread->state = state; if (pim == NULL) thread->pim.result = NFA_PIM_UNUSED; else { copy_pim(&thread->pim, pim); l->has_pim = TRUE; } copy_sub(&thread->subs.norm, &subs->norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub(&thread->subs.synt, &subs->synt); #endif #ifdef ENABLE_LOG report_state("Adding", &thread->subs.norm, state, l->id, pim); did_print = TRUE; #endif } #ifdef ENABLE_LOG if (!did_print) report_state("Processing", &subs->norm, state, l->id, pim); #endif switch (state->c) { case NFA_MATCH: break; case NFA_SPLIT: // order matters here subs = addstate(l, state->out, subs, pim, off_arg); subs = addstate(l, state->out1, subs, pim, off_arg); break; case NFA_EMPTY: case NFA_NOPEN: case NFA_NCLOSE: subs = addstate(l, state->out, subs, pim, off_arg); break; case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif case NFA_ZSTART: if (state->c == NFA_ZSTART) { subidx = 0; sub = &subs->norm; } #ifdef FEAT_SYN_HL else if (state->c >= NFA_ZOPEN && state->c <= NFA_ZOPEN9) { subidx = state->c - NFA_ZOPEN; sub = &subs->synt; } #endif else { subidx = state->c - NFA_MOPEN; sub = &subs->norm; } // avoid compiler warnings save_ptr = NULL; CLEAR_FIELD(save_multipos); // Set the position (with "off" added) in the subexpression. Save // and restore it when it was in use. Otherwise fill any gap. if (REG_MULTI) { if (subidx < sub->in_use) { save_multipos = sub->list.multi[subidx]; save_in_use = -1; } else { save_in_use = sub->in_use; for (i = sub->in_use; i < subidx; ++i) { sub->list.multi[i].start_lnum = -1; sub->list.multi[i].end_lnum = -1; } sub->in_use = subidx + 1; } if (off == -1) { sub->list.multi[subidx].start_lnum = rex.lnum + 1; sub->list.multi[subidx].start_col = 0; } else { sub->list.multi[subidx].start_lnum = rex.lnum; sub->list.multi[subidx].start_col = (colnr_T)(rex.input - rex.line + off); } sub->list.multi[subidx].end_lnum = -1; } else { if (subidx < sub->in_use) { save_ptr = sub->list.line[subidx].start; save_in_use = -1; } else { save_in_use = sub->in_use; for (i = sub->in_use; i < subidx; ++i) { sub->list.line[i].start = NULL; sub->list.line[i].end = NULL; } sub->in_use = subidx + 1; } sub->list.line[subidx].start = rex.input + off; } subs = addstate(l, state->out, subs, pim, off_arg); if (subs == NULL) break; // "subs" may have changed, need to set "sub" again #ifdef FEAT_SYN_HL if (state->c >= NFA_ZOPEN && state->c <= NFA_ZOPEN9) sub = &subs->synt; else #endif sub = &subs->norm; if (save_in_use == -1) { if (REG_MULTI) sub->list.multi[subidx] = save_multipos; else sub->list.line[subidx].start = save_ptr; } else sub->in_use = save_in_use; break; case NFA_MCLOSE: if (rex.nfa_has_zend && (REG_MULTI ? subs->norm.list.multi[0].end_lnum >= 0 : subs->norm.list.line[0].end != NULL)) { // Do not overwrite the position set by \ze. subs = addstate(l, state->out, subs, pim, off_arg); break; } // FALLTHROUGH case NFA_MCLOSE1: case NFA_MCLOSE2: case NFA_MCLOSE3: case NFA_MCLOSE4: case NFA_MCLOSE5: case NFA_MCLOSE6: case NFA_MCLOSE7: case NFA_MCLOSE8: case NFA_MCLOSE9: #ifdef FEAT_SYN_HL case NFA_ZCLOSE: case NFA_ZCLOSE1: case NFA_ZCLOSE2: case NFA_ZCLOSE3: case NFA_ZCLOSE4: case NFA_ZCLOSE5: case NFA_ZCLOSE6: case NFA_ZCLOSE7: case NFA_ZCLOSE8: case NFA_ZCLOSE9: #endif case NFA_ZEND: if (state->c == NFA_ZEND) { subidx = 0; sub = &subs->norm; } #ifdef FEAT_SYN_HL else if (state->c >= NFA_ZCLOSE && state->c <= NFA_ZCLOSE9) { subidx = state->c - NFA_ZCLOSE; sub = &subs->synt; } #endif else { subidx = state->c - NFA_MCLOSE; sub = &subs->norm; } // We don't fill in gaps here, there must have been an MOPEN that // has done that. save_in_use = sub->in_use; if (sub->in_use <= subidx) sub->in_use = subidx + 1; if (REG_MULTI) { save_multipos = sub->list.multi[subidx]; if (off == -1) { sub->list.multi[subidx].end_lnum = rex.lnum + 1; sub->list.multi[subidx].end_col = 0; } else { sub->list.multi[subidx].end_lnum = rex.lnum; sub->list.multi[subidx].end_col = (colnr_T)(rex.input - rex.line + off); } // avoid compiler warnings save_ptr = NULL; } else { save_ptr = sub->list.line[subidx].end; sub->list.line[subidx].end = rex.input + off; // avoid compiler warnings CLEAR_FIELD(save_multipos); } subs = addstate(l, state->out, subs, pim, off_arg); if (subs == NULL) break; // "subs" may have changed, need to set "sub" again #ifdef FEAT_SYN_HL if (state->c >= NFA_ZCLOSE && state->c <= NFA_ZCLOSE9) sub = &subs->synt; else #endif sub = &subs->norm; if (REG_MULTI) sub->list.multi[subidx] = save_multipos; else sub->list.line[subidx].end = save_ptr; sub->in_use = save_in_use; break; } --depth; return subs; } /* * Like addstate(), but the new state(s) are put at position "*ip". * Used for zero-width matches, next state to use is the added one. * This makes sure the order of states to be tried does not change, which * matters for alternatives. */ static regsubs_T * addstate_here( nfa_list_T *l, // runtime state list nfa_state_T *state, // state to update regsubs_T *subs, // pointers to subexpressions nfa_pim_T *pim, // postponed look-behind match int *ip) { int tlen = l->n; int count; int listidx = *ip; regsubs_T *r; // First add the state(s) at the end, so that we know how many there are. // Pass the listidx as offset (avoids adding another argument to // addstate(). r = addstate(l, state, subs, pim, -listidx - ADDSTATE_HERE_OFFSET); if (r == NULL) return NULL; // when "*ip" was at the end of the list, nothing to do if (listidx + 1 == tlen) return r; // re-order to put the new state at the current position count = l->n - tlen; if (count == 0) return r; // no state got added if (count == 1) { // overwrite the current state l->t[listidx] = l->t[l->n - 1]; } else if (count > 1) { if (l->n + count - 1 >= l->len) { // not enough space to move the new states, reallocate the list // and move the states to the right position int newlen = l->len * 3 / 2 + 50; size_t newsize = newlen * sizeof(nfa_thread_T); nfa_thread_T *newl; if ((long)(newsize >> 10) >= p_mmp) { emsg(_(e_maxmempat)); return NULL; } newl = alloc(newsize); if (newl == NULL) return NULL; l->len = newlen; mch_memmove(&(newl[0]), &(l->t[0]), sizeof(nfa_thread_T) * listidx); mch_memmove(&(newl[listidx]), &(l->t[l->n - count]), sizeof(nfa_thread_T) * count); mch_memmove(&(newl[listidx + count]), &(l->t[listidx + 1]), sizeof(nfa_thread_T) * (l->n - count - listidx - 1)); vim_free(l->t); l->t = newl; } else { // make space for new states, then move them from the // end to the current position mch_memmove(&(l->t[listidx + count]), &(l->t[listidx + 1]), sizeof(nfa_thread_T) * (l->n - listidx - 1)); mch_memmove(&(l->t[listidx]), &(l->t[l->n - 1]), sizeof(nfa_thread_T) * count); } } --l->n; *ip = listidx - 1; return r; } /* * Check character class "class" against current character c. */ static int check_char_class(int class, int c) { switch (class) { case NFA_CLASS_ALNUM: if (c >= 1 && c < 128 && isalnum(c)) return OK; break; case NFA_CLASS_ALPHA: if (c >= 1 && c < 128 && isalpha(c)) return OK; break; case NFA_CLASS_BLANK: if (c == ' ' || c == '\t') return OK; break; case NFA_CLASS_CNTRL: if (c >= 1 && c <= 127 && iscntrl(c)) return OK; break; case NFA_CLASS_DIGIT: if (VIM_ISDIGIT(c)) return OK; break; case NFA_CLASS_GRAPH: if (c >= 1 && c <= 127 && isgraph(c)) return OK; break; case NFA_CLASS_LOWER: if (MB_ISLOWER(c) && c != 170 && c != 186) return OK; break; case NFA_CLASS_PRINT: if (vim_isprintc(c)) return OK; break; case NFA_CLASS_PUNCT: if (c >= 1 && c < 128 && ispunct(c)) return OK; break; case NFA_CLASS_SPACE: if ((c >= 9 && c <= 13) || (c == ' ')) return OK; break; case NFA_CLASS_UPPER: if (MB_ISUPPER(c)) return OK; break; case NFA_CLASS_XDIGIT: if (vim_isxdigit(c)) return OK; break; case NFA_CLASS_TAB: if (c == '\t') return OK; break; case NFA_CLASS_RETURN: if (c == '\r') return OK; break; case NFA_CLASS_BACKSPACE: if (c == '\b') return OK; break; case NFA_CLASS_ESCAPE: if (c == '\033') return OK; break; case NFA_CLASS_IDENT: if (vim_isIDc(c)) return OK; break; case NFA_CLASS_KEYWORD: if (reg_iswordc(c)) return OK; break; case NFA_CLASS_FNAME: if (vim_isfilec(c)) return OK; break; default: // should not be here :P siemsg(_(e_ill_char_class), class); return FAIL; } return FAIL; } /* * Check for a match with subexpression "subidx". * Return TRUE if it matches. */ static int match_backref( regsub_T *sub, // pointers to subexpressions int subidx, int *bytelen) // out: length of match in bytes { int len; if (sub->in_use <= subidx) { retempty: // backref was not set, match an empty string *bytelen = 0; return TRUE; } if (REG_MULTI) { if (sub->list.multi[subidx].start_lnum < 0 || sub->list.multi[subidx].end_lnum < 0) goto retempty; if (sub->list.multi[subidx].start_lnum == rex.lnum && sub->list.multi[subidx].end_lnum == rex.lnum) { len = sub->list.multi[subidx].end_col - sub->list.multi[subidx].start_col; if (cstrncmp(rex.line + sub->list.multi[subidx].start_col, rex.input, &len) == 0) { *bytelen = len; return TRUE; } } else { if (match_with_backref( sub->list.multi[subidx].start_lnum, sub->list.multi[subidx].start_col, sub->list.multi[subidx].end_lnum, sub->list.multi[subidx].end_col, bytelen) == RA_MATCH) return TRUE; } } else { if (sub->list.line[subidx].start == NULL || sub->list.line[subidx].end == NULL) goto retempty; len = (int)(sub->list.line[subidx].end - sub->list.line[subidx].start); if (cstrncmp(sub->list.line[subidx].start, rex.input, &len) == 0) { *bytelen = len; return TRUE; } } return FALSE; } #ifdef FEAT_SYN_HL /* * Check for a match with \z subexpression "subidx". * Return TRUE if it matches. */ static int match_zref( int subidx, int *bytelen) // out: length of match in bytes { int len; cleanup_zsubexpr(); if (re_extmatch_in == NULL || re_extmatch_in->matches[subidx] == NULL) { // backref was not set, match an empty string *bytelen = 0; return TRUE; } len = (int)STRLEN(re_extmatch_in->matches[subidx]); if (cstrncmp(re_extmatch_in->matches[subidx], rex.input, &len) == 0) { *bytelen = len; return TRUE; } return FALSE; } #endif /* * Save list IDs for all NFA states of "prog" into "list". * Also reset the IDs to zero. * Only used for the recursive value lastlist[1]. */ static void nfa_save_listids(nfa_regprog_T *prog, int *list) { int i; nfa_state_T *p; // Order in the list is reverse, it's a bit faster that way. p = &prog->state[0]; for (i = prog->nstate; --i >= 0; ) { list[i] = p->lastlist[1]; p->lastlist[1] = 0; ++p; } } /* * Restore list IDs from "list" to all NFA states. */ static void nfa_restore_listids(nfa_regprog_T *prog, int *list) { int i; nfa_state_T *p; p = &prog->state[0]; for (i = prog->nstate; --i >= 0; ) { p->lastlist[1] = list[i]; ++p; } } static int nfa_re_num_cmp(long_u val, int op, long_u pos) { if (op == 1) return pos > val; if (op == 2) return pos < val; return val == pos; } static int nfa_regmatch(nfa_regprog_T *prog, nfa_state_T *start, regsubs_T *submatch, regsubs_T *m); /* * Recursively call nfa_regmatch() * "pim" is NULL or contains info about a Postponed Invisible Match (start * position). */ static int recursive_regmatch( nfa_state_T *state, nfa_pim_T *pim, nfa_regprog_T *prog, regsubs_T *submatch, regsubs_T *m, int **listids, int *listids_len) { int save_reginput_col = (int)(rex.input - rex.line); int save_reglnum = rex.lnum; int save_nfa_match = nfa_match; int save_nfa_listid = rex.nfa_listid; save_se_T *save_nfa_endp = nfa_endp; save_se_T endpos; save_se_T *endposp = NULL; int result; int need_restore = FALSE; if (pim != NULL) { // start at the position where the postponed match was if (REG_MULTI) rex.input = rex.line + pim->end.pos.col; else rex.input = pim->end.ptr; } if (state->c == NFA_START_INVISIBLE_BEFORE || state->c == NFA_START_INVISIBLE_BEFORE_FIRST || state->c == NFA_START_INVISIBLE_BEFORE_NEG || state->c == NFA_START_INVISIBLE_BEFORE_NEG_FIRST) { // The recursive match must end at the current position. When "pim" is // not NULL it specifies the current position. endposp = &endpos; if (REG_MULTI) { if (pim == NULL) { endpos.se_u.pos.col = (int)(rex.input - rex.line); endpos.se_u.pos.lnum = rex.lnum; } else endpos.se_u.pos = pim->end.pos; } else { if (pim == NULL) endpos.se_u.ptr = rex.input; else endpos.se_u.ptr = pim->end.ptr; } // Go back the specified number of bytes, or as far as the // start of the previous line, to try matching "\@<=" or // not matching "\@<!". This is very inefficient, limit the number of // bytes if possible. if (state->val <= 0) { if (REG_MULTI) { rex.line = reg_getline(--rex.lnum); if (rex.line == NULL) // can't go before the first line rex.line = reg_getline(++rex.lnum); } rex.input = rex.line; } else { if (REG_MULTI && (int)(rex.input - rex.line) < state->val) { // Not enough bytes in this line, go to end of // previous line. rex.line = reg_getline(--rex.lnum); if (rex.line == NULL) { // can't go before the first line rex.line = reg_getline(++rex.lnum); rex.input = rex.line; } else rex.input = rex.line + STRLEN(rex.line); } if ((int)(rex.input - rex.line) >= state->val) { rex.input -= state->val; if (has_mbyte) rex.input -= mb_head_off(rex.line, rex.input); } else rex.input = rex.line; } } #ifdef ENABLE_LOG if (log_fd != stderr) fclose(log_fd); log_fd = NULL; #endif // Have to clear the lastlist field of the NFA nodes, so that // nfa_regmatch() and addstate() can run properly after recursion. if (nfa_ll_index == 1) { // Already calling nfa_regmatch() recursively. Save the lastlist[1] // values and clear them. if (*listids == NULL || *listids_len < prog->nstate) { vim_free(*listids); *listids = ALLOC_MULT(int, prog->nstate); if (*listids == NULL) { emsg(_("E878: (NFA) Could not allocate memory for branch traversal!")); return 0; } *listids_len = prog->nstate; } nfa_save_listids(prog, *listids); need_restore = TRUE; // any value of rex.nfa_listid will do } else { // First recursive nfa_regmatch() call, switch to the second lastlist // entry. Make sure rex.nfa_listid is different from a previous // recursive call, because some states may still have this ID. ++nfa_ll_index; if (rex.nfa_listid <= rex.nfa_alt_listid) rex.nfa_listid = rex.nfa_alt_listid; } // Call nfa_regmatch() to check if the current concat matches at this // position. The concat ends with the node NFA_END_INVISIBLE nfa_endp = endposp; result = nfa_regmatch(prog, state->out, submatch, m); if (need_restore) nfa_restore_listids(prog, *listids); else { --nfa_ll_index; rex.nfa_alt_listid = rex.nfa_listid; } // restore position in input text rex.lnum = save_reglnum; if (REG_MULTI) rex.line = reg_getline(rex.lnum); rex.input = rex.line + save_reginput_col; if (result != NFA_TOO_EXPENSIVE) { nfa_match = save_nfa_match; rex.nfa_listid = save_nfa_listid; } nfa_endp = save_nfa_endp; #ifdef ENABLE_LOG log_fd = fopen(NFA_REGEXP_RUN_LOG, "a"); if (log_fd != NULL) { fprintf(log_fd, "****************************\n"); fprintf(log_fd, "FINISHED RUNNING nfa_regmatch() recursively\n"); fprintf(log_fd, "MATCH = %s\n", result == TRUE ? "OK" : "FALSE"); fprintf(log_fd, "****************************\n"); } else { emsg(_(e_log_open_failed)); log_fd = stderr; } #endif return result; } /* * Estimate the chance of a match with "state" failing. * empty match: 0 * NFA_ANY: 1 * specific character: 99 */ static int failure_chance(nfa_state_T *state, int depth) { int c = state->c; int l, r; // detect looping if (depth > 4) return 1; switch (c) { case NFA_SPLIT: if (state->out->c == NFA_SPLIT || state->out1->c == NFA_SPLIT) // avoid recursive stuff return 1; // two alternatives, use the lowest failure chance l = failure_chance(state->out, depth + 1); r = failure_chance(state->out1, depth + 1); return l < r ? l : r; case NFA_ANY: // matches anything, unlikely to fail return 1; case NFA_MATCH: case NFA_MCLOSE: case NFA_ANY_COMPOSING: // empty match works always return 0; case NFA_START_INVISIBLE: case NFA_START_INVISIBLE_FIRST: case NFA_START_INVISIBLE_NEG: case NFA_START_INVISIBLE_NEG_FIRST: case NFA_START_INVISIBLE_BEFORE: case NFA_START_INVISIBLE_BEFORE_FIRST: case NFA_START_INVISIBLE_BEFORE_NEG: case NFA_START_INVISIBLE_BEFORE_NEG_FIRST: case NFA_START_PATTERN: // recursive regmatch is expensive, use low failure chance return 5; case NFA_BOL: case NFA_EOL: case NFA_BOF: case NFA_EOF: case NFA_NEWL: return 99; case NFA_BOW: case NFA_EOW: return 90; case NFA_MOPEN: case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: case NFA_ZCLOSE: case NFA_ZCLOSE1: case NFA_ZCLOSE2: case NFA_ZCLOSE3: case NFA_ZCLOSE4: case NFA_ZCLOSE5: case NFA_ZCLOSE6: case NFA_ZCLOSE7: case NFA_ZCLOSE8: case NFA_ZCLOSE9: #endif case NFA_NOPEN: case NFA_MCLOSE1: case NFA_MCLOSE2: case NFA_MCLOSE3: case NFA_MCLOSE4: case NFA_MCLOSE5: case NFA_MCLOSE6: case NFA_MCLOSE7: case NFA_MCLOSE8: case NFA_MCLOSE9: case NFA_NCLOSE: return failure_chance(state->out, depth + 1); case NFA_BACKREF1: case NFA_BACKREF2: case NFA_BACKREF3: case NFA_BACKREF4: case NFA_BACKREF5: case NFA_BACKREF6: case NFA_BACKREF7: case NFA_BACKREF8: case NFA_BACKREF9: #ifdef FEAT_SYN_HL case NFA_ZREF1: case NFA_ZREF2: case NFA_ZREF3: case NFA_ZREF4: case NFA_ZREF5: case NFA_ZREF6: case NFA_ZREF7: case NFA_ZREF8: case NFA_ZREF9: #endif // backreferences don't match in many places return 94; case NFA_LNUM_GT: case NFA_LNUM_LT: case NFA_COL_GT: case NFA_COL_LT: case NFA_VCOL_GT: case NFA_VCOL_LT: case NFA_MARK_GT: case NFA_MARK_LT: case NFA_VISUAL: // before/after positions don't match very often return 85; case NFA_LNUM: return 90; case NFA_CURSOR: case NFA_COL: case NFA_VCOL: case NFA_MARK: // specific positions rarely match return 98; case NFA_COMPOSING: return 95; default: if (c > 0) // character match fails often return 95; } // something else, includes character classes return 50; } /* * Skip until the char "c" we know a match must start with. */ static int skip_to_start(int c, colnr_T *colp) { char_u *s; // Used often, do some work to avoid call overhead. if (!rex.reg_ic && !has_mbyte) s = vim_strbyte(rex.line + *colp, c); else s = cstrchr(rex.line + *colp, c); if (s == NULL) return FAIL; *colp = (int)(s - rex.line); return OK; } /* * Check for a match with match_text. * Called after skip_to_start() has found regstart. * Returns zero for no match, 1 for a match. */ static long find_match_text(colnr_T startcol, int regstart, char_u *match_text) { colnr_T col = startcol; int c1, c2; int len1, len2; int match; for (;;) { match = TRUE; len2 = MB_CHAR2LEN(regstart); // skip regstart for (len1 = 0; match_text[len1] != NUL; len1 += MB_CHAR2LEN(c1)) { c1 = PTR2CHAR(match_text + len1); c2 = PTR2CHAR(rex.line + col + len2); if (c1 != c2 && (!rex.reg_ic || MB_CASEFOLD(c1) != MB_CASEFOLD(c2))) { match = FALSE; break; } len2 += MB_CHAR2LEN(c2); } if (match // check that no composing char follows && !(enc_utf8 && utf_iscomposing(PTR2CHAR(rex.line + col + len2)))) { cleanup_subexpr(); if (REG_MULTI) { rex.reg_startpos[0].lnum = rex.lnum; rex.reg_startpos[0].col = col; rex.reg_endpos[0].lnum = rex.lnum; rex.reg_endpos[0].col = col + len2; } else { rex.reg_startp[0] = rex.line + col; rex.reg_endp[0] = rex.line + col + len2; } return 1L; } // Try finding regstart after the current match. col += MB_CHAR2LEN(regstart); // skip regstart if (skip_to_start(regstart, &col) == FAIL) break; } return 0L; } #ifdef FEAT_RELTIME static int nfa_did_time_out() { if (nfa_time_limit != NULL && profile_passed_limit(nfa_time_limit)) { if (nfa_timed_out != NULL) *nfa_timed_out = TRUE; return TRUE; } return FALSE; } #endif /* * Main matching routine. * * Run NFA to determine whether it matches rex.input. * * When "nfa_endp" is not NULL it is a required end-of-match position. * * Return TRUE if there is a match, FALSE if there is no match, * NFA_TOO_EXPENSIVE if we end up with too many states. * When there is a match "submatch" contains the positions. * * Note: Caller must ensure that: start != NULL. */ static int nfa_regmatch( nfa_regprog_T *prog, nfa_state_T *start, regsubs_T *submatch, regsubs_T *m) { int result = FALSE; size_t size = 0; int flag = 0; int go_to_nextline = FALSE; nfa_thread_T *t; nfa_list_T list[2]; int listidx; nfa_list_T *thislist; nfa_list_T *nextlist; int *listids = NULL; int listids_len = 0; nfa_state_T *add_state; int add_here; int add_count; int add_off = 0; int toplevel = start->c == NFA_MOPEN; regsubs_T *r; #ifdef NFA_REGEXP_DEBUG_LOG FILE *debug; #endif // Some patterns may take a long time to match, especially when using // recursive_regmatch(). Allow interrupting them with CTRL-C. fast_breakcheck(); if (got_int) return FALSE; #ifdef FEAT_RELTIME if (nfa_did_time_out()) return FALSE; #endif #ifdef NFA_REGEXP_DEBUG_LOG debug = fopen(NFA_REGEXP_DEBUG_LOG, "a"); if (debug == NULL) { semsg("(NFA) COULD NOT OPEN %s!", NFA_REGEXP_DEBUG_LOG); return FALSE; } #endif nfa_match = FALSE; // Allocate memory for the lists of nodes. size = (prog->nstate + 1) * sizeof(nfa_thread_T); list[0].t = alloc(size); list[0].len = prog->nstate + 1; list[1].t = alloc(size); list[1].len = prog->nstate + 1; if (list[0].t == NULL || list[1].t == NULL) goto theend; #ifdef ENABLE_LOG log_fd = fopen(NFA_REGEXP_RUN_LOG, "a"); if (log_fd != NULL) { fprintf(log_fd, "**********************************\n"); nfa_set_code(start->c); fprintf(log_fd, " RUNNING nfa_regmatch() starting with state %d, code %s\n", abs(start->id), code); fprintf(log_fd, "**********************************\n"); } else { emsg(_(e_log_open_failed)); log_fd = stderr; } #endif thislist = &list[0]; thislist->n = 0; thislist->has_pim = FALSE; nextlist = &list[1]; nextlist->n = 0; nextlist->has_pim = FALSE; #ifdef ENABLE_LOG fprintf(log_fd, "(---) STARTSTATE first\n"); #endif thislist->id = rex.nfa_listid + 1; // Inline optimized code for addstate(thislist, start, m, 0) if we know // it's the first MOPEN. if (toplevel) { if (REG_MULTI) { m->norm.list.multi[0].start_lnum = rex.lnum; m->norm.list.multi[0].start_col = (colnr_T)(rex.input - rex.line); } else m->norm.list.line[0].start = rex.input; m->norm.in_use = 1; r = addstate(thislist, start->out, m, NULL, 0); } else r = addstate(thislist, start, m, NULL, 0); if (r == NULL) { nfa_match = NFA_TOO_EXPENSIVE; goto theend; } #define ADD_STATE_IF_MATCH(state) \ if (result) { \ add_state = state->out; \ add_off = clen; \ } /* * Run for each character. */ for (;;) { int curc; int clen; if (has_mbyte) { curc = (*mb_ptr2char)(rex.input); clen = (*mb_ptr2len)(rex.input); } else { curc = *rex.input; clen = 1; } if (curc == NUL) { clen = 0; go_to_nextline = FALSE; } // swap lists thislist = &list[flag]; nextlist = &list[flag ^= 1]; nextlist->n = 0; // clear nextlist nextlist->has_pim = FALSE; ++rex.nfa_listid; if (prog->re_engine == AUTOMATIC_ENGINE && (rex.nfa_listid >= NFA_MAX_STATES # ifdef FEAT_EVAL || nfa_fail_for_testing # endif )) { // too many states, retry with old engine nfa_match = NFA_TOO_EXPENSIVE; goto theend; } thislist->id = rex.nfa_listid; nextlist->id = rex.nfa_listid + 1; #ifdef ENABLE_LOG fprintf(log_fd, "------------------------------------------\n"); fprintf(log_fd, ">>> Reginput is \"%s\"\n", rex.input); fprintf(log_fd, ">>> Advanced one character... Current char is %c (code %d) \n", curc, (int)curc); fprintf(log_fd, ">>> Thislist has %d states available: ", thislist->n); { int i; for (i = 0; i < thislist->n; i++) fprintf(log_fd, "%d ", abs(thislist->t[i].state->id)); } fprintf(log_fd, "\n"); #endif #ifdef NFA_REGEXP_DEBUG_LOG fprintf(debug, "\n-------------------\n"); #endif /* * If the state lists are empty we can stop. */ if (thislist->n == 0) break; // compute nextlist for (listidx = 0; listidx < thislist->n; ++listidx) { // If the list gets very long there probably is something wrong. // At least allow interrupting with CTRL-C. fast_breakcheck(); if (got_int) break; #ifdef FEAT_RELTIME if (nfa_time_limit != NULL && ++nfa_time_count == 20) { nfa_time_count = 0; if (nfa_did_time_out()) break; } #endif t = &thislist->t[listidx]; #ifdef NFA_REGEXP_DEBUG_LOG nfa_set_code(t->state->c); fprintf(debug, "%s, ", code); #endif #ifdef ENABLE_LOG { int col; if (t->subs.norm.in_use <= 0) col = -1; else if (REG_MULTI) col = t->subs.norm.list.multi[0].start_col; else col = (int)(t->subs.norm.list.line[0].start - rex.line); nfa_set_code(t->state->c); fprintf(log_fd, "(%d) char %d %s (start col %d)%s... \n", abs(t->state->id), (int)t->state->c, code, col, pim_info(&t->pim)); } #endif /* * Handle the possible codes of the current state. * The most important is NFA_MATCH. */ add_state = NULL; add_here = FALSE; add_count = 0; switch (t->state->c) { case NFA_MATCH: { // If the match is not at the start of the line, ends before a // composing characters and rex.reg_icombine is not set, that // is not really a match. if (enc_utf8 && !rex.reg_icombine && rex.input != rex.line && utf_iscomposing(curc)) break; nfa_match = TRUE; copy_sub(&submatch->norm, &t->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub(&submatch->synt, &t->subs.synt); #endif #ifdef ENABLE_LOG log_subsexpr(&t->subs); #endif // Found the left-most longest match, do not look at any other // states at this position. When the list of states is going // to be empty quit without advancing, so that "rex.input" is // correct. if (nextlist->n == 0) clen = 0; goto nextchar; } case NFA_END_INVISIBLE: case NFA_END_INVISIBLE_NEG: case NFA_END_PATTERN: /* * This is only encountered after a NFA_START_INVISIBLE or * NFA_START_INVISIBLE_BEFORE node. * They surround a zero-width group, used with "\@=", "\&", * "\@!", "\@<=" and "\@<!". * If we got here, it means that the current "invisible" group * finished successfully, so return control to the parent * nfa_regmatch(). For a look-behind match only when it ends * in the position in "nfa_endp". * Submatches are stored in *m, and used in the parent call. */ #ifdef ENABLE_LOG if (nfa_endp != NULL) { if (REG_MULTI) fprintf(log_fd, "Current lnum: %d, endp lnum: %d; current col: %d, endp col: %d\n", (int)rex.lnum, (int)nfa_endp->se_u.pos.lnum, (int)(rex.input - rex.line), nfa_endp->se_u.pos.col); else fprintf(log_fd, "Current col: %d, endp col: %d\n", (int)(rex.input - rex.line), (int)(nfa_endp->se_u.ptr - rex.input)); } #endif // If "nfa_endp" is set it's only a match if it ends at // "nfa_endp" if (nfa_endp != NULL && (REG_MULTI ? (rex.lnum != nfa_endp->se_u.pos.lnum || (int)(rex.input - rex.line) != nfa_endp->se_u.pos.col) : rex.input != nfa_endp->se_u.ptr)) break; // do not set submatches for \@! if (t->state->c != NFA_END_INVISIBLE_NEG) { copy_sub(&m->norm, &t->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub(&m->synt, &t->subs.synt); #endif } #ifdef ENABLE_LOG fprintf(log_fd, "Match found:\n"); log_subsexpr(m); #endif nfa_match = TRUE; // See comment above at "goto nextchar". if (nextlist->n == 0) clen = 0; goto nextchar; case NFA_START_INVISIBLE: case NFA_START_INVISIBLE_FIRST: case NFA_START_INVISIBLE_NEG: case NFA_START_INVISIBLE_NEG_FIRST: case NFA_START_INVISIBLE_BEFORE: case NFA_START_INVISIBLE_BEFORE_FIRST: case NFA_START_INVISIBLE_BEFORE_NEG: case NFA_START_INVISIBLE_BEFORE_NEG_FIRST: { #ifdef ENABLE_LOG fprintf(log_fd, "Failure chance invisible: %d, what follows: %d\n", failure_chance(t->state->out, 0), failure_chance(t->state->out1->out, 0)); #endif // Do it directly if there already is a PIM or when // nfa_postprocess() detected it will work better. if (t->pim.result != NFA_PIM_UNUSED || t->state->c == NFA_START_INVISIBLE_FIRST || t->state->c == NFA_START_INVISIBLE_NEG_FIRST || t->state->c == NFA_START_INVISIBLE_BEFORE_FIRST || t->state->c == NFA_START_INVISIBLE_BEFORE_NEG_FIRST) { int in_use = m->norm.in_use; // Copy submatch info for the recursive call, opposite // of what happens on success below. copy_sub_off(&m->norm, &t->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&m->synt, &t->subs.synt); #endif /* * First try matching the invisible match, then what * follows. */ result = recursive_regmatch(t->state, NULL, prog, submatch, m, &listids, &listids_len); if (result == NFA_TOO_EXPENSIVE) { nfa_match = result; goto theend; } // for \@! and \@<! it is a match when the result is // FALSE if (result != (t->state->c == NFA_START_INVISIBLE_NEG || t->state->c == NFA_START_INVISIBLE_NEG_FIRST || t->state->c == NFA_START_INVISIBLE_BEFORE_NEG || t->state->c == NFA_START_INVISIBLE_BEFORE_NEG_FIRST)) { // Copy submatch info from the recursive call copy_sub_off(&t->subs.norm, &m->norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&t->subs.synt, &m->synt); #endif // If the pattern has \ze and it matched in the // sub pattern, use it. copy_ze_off(&t->subs.norm, &m->norm); // t->state->out1 is the corresponding // END_INVISIBLE node; Add its out to the current // list (zero-width match). add_here = TRUE; add_state = t->state->out1->out; } m->norm.in_use = in_use; } else { nfa_pim_T pim; /* * First try matching what follows. Only if a match * is found verify the invisible match matches. Add a * nfa_pim_T to the following states, it contains info * about the invisible match. */ pim.state = t->state; pim.result = NFA_PIM_TODO; pim.subs.norm.in_use = 0; #ifdef FEAT_SYN_HL pim.subs.synt.in_use = 0; #endif if (REG_MULTI) { pim.end.pos.col = (int)(rex.input - rex.line); pim.end.pos.lnum = rex.lnum; } else pim.end.ptr = rex.input; // t->state->out1 is the corresponding END_INVISIBLE // node; Add its out to the current list (zero-width // match). if (addstate_here(thislist, t->state->out1->out, &t->subs, &pim, &listidx) == NULL) { nfa_match = NFA_TOO_EXPENSIVE; goto theend; } } } break; case NFA_START_PATTERN: { nfa_state_T *skip = NULL; #ifdef ENABLE_LOG int skip_lid = 0; #endif // There is no point in trying to match the pattern if the // output state is not going to be added to the list. if (state_in_list(nextlist, t->state->out1->out, &t->subs)) { skip = t->state->out1->out; #ifdef ENABLE_LOG skip_lid = nextlist->id; #endif } else if (state_in_list(nextlist, t->state->out1->out->out, &t->subs)) { skip = t->state->out1->out->out; #ifdef ENABLE_LOG skip_lid = nextlist->id; #endif } else if (state_in_list(thislist, t->state->out1->out->out, &t->subs)) { skip = t->state->out1->out->out; #ifdef ENABLE_LOG skip_lid = thislist->id; #endif } if (skip != NULL) { #ifdef ENABLE_LOG nfa_set_code(skip->c); fprintf(log_fd, "> Not trying to match pattern, output state %d is already in list %d. char %d: %s\n", abs(skip->id), skip_lid, skip->c, code); #endif break; } // Copy submatch info to the recursive call, opposite of what // happens afterwards. copy_sub_off(&m->norm, &t->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&m->synt, &t->subs.synt); #endif // First try matching the pattern. result = recursive_regmatch(t->state, NULL, prog, submatch, m, &listids, &listids_len); if (result == NFA_TOO_EXPENSIVE) { nfa_match = result; goto theend; } if (result) { int bytelen; #ifdef ENABLE_LOG fprintf(log_fd, "NFA_START_PATTERN matches:\n"); log_subsexpr(m); #endif // Copy submatch info from the recursive call copy_sub_off(&t->subs.norm, &m->norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&t->subs.synt, &m->synt); #endif // Now we need to skip over the matched text and then // continue with what follows. if (REG_MULTI) // TODO: multi-line match bytelen = m->norm.list.multi[0].end_col - (int)(rex.input - rex.line); else bytelen = (int)(m->norm.list.line[0].end - rex.input); #ifdef ENABLE_LOG fprintf(log_fd, "NFA_START_PATTERN length: %d\n", bytelen); #endif if (bytelen == 0) { // empty match, output of corresponding // NFA_END_PATTERN/NFA_SKIP to be used at current // position add_here = TRUE; add_state = t->state->out1->out->out; } else if (bytelen <= clen) { // match current character, output of corresponding // NFA_END_PATTERN to be used at next position. add_state = t->state->out1->out->out; add_off = clen; } else { // skip over the matched characters, set character // count in NFA_SKIP add_state = t->state->out1->out; add_off = bytelen; add_count = bytelen - clen; } } break; } case NFA_BOL: if (rex.input == rex.line) { add_here = TRUE; add_state = t->state->out; } break; case NFA_EOL: if (curc == NUL) { add_here = TRUE; add_state = t->state->out; } break; case NFA_BOW: result = TRUE; if (curc == NUL) result = FALSE; else if (has_mbyte) { int this_class; // Get class of current and previous char (if it exists). this_class = mb_get_class_buf(rex.input, rex.reg_buf); if (this_class <= 1) result = FALSE; else if (reg_prev_class() == this_class) result = FALSE; } else if (!vim_iswordc_buf(curc, rex.reg_buf) || (rex.input > rex.line && vim_iswordc_buf(rex.input[-1], rex.reg_buf))) result = FALSE; if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_EOW: result = TRUE; if (rex.input == rex.line) result = FALSE; else if (has_mbyte) { int this_class, prev_class; // Get class of current and previous char (if it exists). this_class = mb_get_class_buf(rex.input, rex.reg_buf); prev_class = reg_prev_class(); if (this_class == prev_class || prev_class == 0 || prev_class == 1) result = FALSE; } else if (!vim_iswordc_buf(rex.input[-1], rex.reg_buf) || (rex.input[0] != NUL && vim_iswordc_buf(curc, rex.reg_buf))) result = FALSE; if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_BOF: if (rex.lnum == 0 && rex.input == rex.line && (!REG_MULTI || rex.reg_firstlnum == 1)) { add_here = TRUE; add_state = t->state->out; } break; case NFA_EOF: if (rex.lnum == rex.reg_maxline && curc == NUL) { add_here = TRUE; add_state = t->state->out; } break; case NFA_COMPOSING: { int mc = curc; int len = 0; nfa_state_T *end; nfa_state_T *sta; int cchars[MAX_MCO]; int ccount = 0; int j; sta = t->state->out; len = 0; if (utf_iscomposing(sta->c)) { // Only match composing character(s), ignore base // character. Used for ".{composing}" and "{composing}" // (no preceding character). len += mb_char2len(mc); } if (rex.reg_icombine && len == 0) { // If \Z was present, then ignore composing characters. // When ignoring the base character this always matches. if (sta->c != curc) result = FAIL; else result = OK; while (sta->c != NFA_END_COMPOSING) sta = sta->out; } // Check base character matches first, unless ignored. else if (len > 0 || mc == sta->c) { if (len == 0) { len += mb_char2len(mc); sta = sta->out; } // We don't care about the order of composing characters. // Get them into cchars[] first. while (len < clen) { mc = mb_ptr2char(rex.input + len); cchars[ccount++] = mc; len += mb_char2len(mc); if (ccount == MAX_MCO) break; } // Check that each composing char in the pattern matches a // composing char in the text. We do not check if all // composing chars are matched. result = OK; while (sta->c != NFA_END_COMPOSING) { for (j = 0; j < ccount; ++j) if (cchars[j] == sta->c) break; if (j == ccount) { result = FAIL; break; } sta = sta->out; } } else result = FAIL; end = t->state->out1; // NFA_END_COMPOSING ADD_STATE_IF_MATCH(end); break; } case NFA_NEWL: if (curc == NUL && !rex.reg_line_lbr && REG_MULTI && rex.lnum <= rex.reg_maxline) { go_to_nextline = TRUE; // Pass -1 for the offset, which means taking the position // at the start of the next line. add_state = t->state->out; add_off = -1; } else if (curc == '\n' && rex.reg_line_lbr) { // match \n as if it is an ordinary character add_state = t->state->out; add_off = 1; } break; case NFA_START_COLL: case NFA_START_NEG_COLL: { // What follows is a list of characters, until NFA_END_COLL. // One of them must match or none of them must match. nfa_state_T *state; int result_if_matched; int c1, c2; // Never match EOL. If it's part of the collection it is added // as a separate state with an OR. if (curc == NUL) break; state = t->state->out; result_if_matched = (t->state->c == NFA_START_COLL); for (;;) { if (state->c == NFA_END_COLL) { result = !result_if_matched; break; } if (state->c == NFA_RANGE_MIN) { c1 = state->val; state = state->out; // advance to NFA_RANGE_MAX c2 = state->val; #ifdef ENABLE_LOG fprintf(log_fd, "NFA_RANGE_MIN curc=%d c1=%d c2=%d\n", curc, c1, c2); #endif if (curc >= c1 && curc <= c2) { result = result_if_matched; break; } if (rex.reg_ic) { int curc_low = MB_CASEFOLD(curc); int done = FALSE; for ( ; c1 <= c2; ++c1) if (MB_CASEFOLD(c1) == curc_low) { result = result_if_matched; done = TRUE; break; } if (done) break; } } else if (state->c < 0 ? check_char_class(state->c, curc) : (curc == state->c || (rex.reg_ic && MB_CASEFOLD(curc) == MB_CASEFOLD(state->c)))) { result = result_if_matched; break; } state = state->out; } if (result) { // next state is in out of the NFA_END_COLL, out1 of // START points to the END state add_state = t->state->out1->out; add_off = clen; } break; } case NFA_ANY: // Any char except '\0', (end of input) does not match. if (curc > 0) { add_state = t->state->out; add_off = clen; } break; case NFA_ANY_COMPOSING: // On a composing character skip over it. Otherwise do // nothing. Always matches. if (enc_utf8 && utf_iscomposing(curc)) { add_off = clen; } else { add_here = TRUE; add_off = 0; } add_state = t->state->out; break; /* * Character classes like \a for alpha, \d for digit etc. */ case NFA_IDENT: // \i result = vim_isIDc(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_SIDENT: // \I result = !VIM_ISDIGIT(curc) && vim_isIDc(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_KWORD: // \k result = vim_iswordp_buf(rex.input, rex.reg_buf); ADD_STATE_IF_MATCH(t->state); break; case NFA_SKWORD: // \K result = !VIM_ISDIGIT(curc) && vim_iswordp_buf(rex.input, rex.reg_buf); ADD_STATE_IF_MATCH(t->state); break; case NFA_FNAME: // \f result = vim_isfilec(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_SFNAME: // \F result = !VIM_ISDIGIT(curc) && vim_isfilec(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_PRINT: // \p result = vim_isprintc(PTR2CHAR(rex.input)); ADD_STATE_IF_MATCH(t->state); break; case NFA_SPRINT: // \P result = !VIM_ISDIGIT(curc) && vim_isprintc(PTR2CHAR(rex.input)); ADD_STATE_IF_MATCH(t->state); break; case NFA_WHITE: // \s result = VIM_ISWHITE(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NWHITE: // \S result = curc != NUL && !VIM_ISWHITE(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_DIGIT: // \d result = ri_digit(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NDIGIT: // \D result = curc != NUL && !ri_digit(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_HEX: // \x result = ri_hex(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NHEX: // \X result = curc != NUL && !ri_hex(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_OCTAL: // \o result = ri_octal(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NOCTAL: // \O result = curc != NUL && !ri_octal(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_WORD: // \w result = ri_word(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NWORD: // \W result = curc != NUL && !ri_word(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_HEAD: // \h result = ri_head(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NHEAD: // \H result = curc != NUL && !ri_head(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_ALPHA: // \a result = ri_alpha(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NALPHA: // \A result = curc != NUL && !ri_alpha(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_LOWER: // \l result = ri_lower(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NLOWER: // \L result = curc != NUL && !ri_lower(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_UPPER: // \u result = ri_upper(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_NUPPER: // \U result = curc != NUL && !ri_upper(curc); ADD_STATE_IF_MATCH(t->state); break; case NFA_LOWER_IC: // [a-z] result = ri_lower(curc) || (rex.reg_ic && ri_upper(curc)); ADD_STATE_IF_MATCH(t->state); break; case NFA_NLOWER_IC: // [^a-z] result = curc != NUL && !(ri_lower(curc) || (rex.reg_ic && ri_upper(curc))); ADD_STATE_IF_MATCH(t->state); break; case NFA_UPPER_IC: // [A-Z] result = ri_upper(curc) || (rex.reg_ic && ri_lower(curc)); ADD_STATE_IF_MATCH(t->state); break; case NFA_NUPPER_IC: // ^[A-Z] result = curc != NUL && !(ri_upper(curc) || (rex.reg_ic && ri_lower(curc))); ADD_STATE_IF_MATCH(t->state); break; case NFA_BACKREF1: case NFA_BACKREF2: case NFA_BACKREF3: case NFA_BACKREF4: case NFA_BACKREF5: case NFA_BACKREF6: case NFA_BACKREF7: case NFA_BACKREF8: case NFA_BACKREF9: #ifdef FEAT_SYN_HL case NFA_ZREF1: case NFA_ZREF2: case NFA_ZREF3: case NFA_ZREF4: case NFA_ZREF5: case NFA_ZREF6: case NFA_ZREF7: case NFA_ZREF8: case NFA_ZREF9: #endif // \1 .. \9 \z1 .. \z9 { int subidx; int bytelen; if (t->state->c <= NFA_BACKREF9) { subidx = t->state->c - NFA_BACKREF1 + 1; result = match_backref(&t->subs.norm, subidx, &bytelen); } #ifdef FEAT_SYN_HL else { subidx = t->state->c - NFA_ZREF1 + 1; result = match_zref(subidx, &bytelen); } #endif if (result) { if (bytelen == 0) { // empty match always works, output of NFA_SKIP to be // used next add_here = TRUE; add_state = t->state->out->out; } else if (bytelen <= clen) { // match current character, jump ahead to out of // NFA_SKIP add_state = t->state->out->out; add_off = clen; } else { // skip over the matched characters, set character // count in NFA_SKIP add_state = t->state->out; add_off = bytelen; add_count = bytelen - clen; } } break; } case NFA_SKIP: // character of previous matching \1 .. \9 or \@> if (t->count - clen <= 0) { // end of match, go to what follows add_state = t->state->out; add_off = clen; } else { // add state again with decremented count add_state = t->state; add_off = 0; add_count = t->count - clen; } break; case NFA_LNUM: case NFA_LNUM_GT: case NFA_LNUM_LT: result = (REG_MULTI && nfa_re_num_cmp(t->state->val, t->state->c - NFA_LNUM, (long_u)(rex.lnum + rex.reg_firstlnum))); if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_COL: case NFA_COL_GT: case NFA_COL_LT: result = nfa_re_num_cmp(t->state->val, t->state->c - NFA_COL, (long_u)(rex.input - rex.line) + 1); if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_VCOL: case NFA_VCOL_GT: case NFA_VCOL_LT: { int op = t->state->c - NFA_VCOL; colnr_T col = (colnr_T)(rex.input - rex.line); win_T *wp = rex.reg_win == NULL ? curwin : rex.reg_win; // Bail out quickly when there can't be a match, avoid the // overhead of win_linetabsize() on long lines. if (op != 1 && col > t->state->val * (has_mbyte ? MB_MAXBYTES : 1)) break; result = FALSE; if (op == 1 && col - 1 > t->state->val && col > 100) { int ts = wp->w_buffer->b_p_ts; // Guess that a character won't use more columns than // 'tabstop', with a minimum of 4. if (ts < 4) ts = 4; result = col > t->state->val * ts; } if (!result) result = nfa_re_num_cmp(t->state->val, op, (long_u)win_linetabsize(wp, rex.line, col) + 1); if (result) { add_here = TRUE; add_state = t->state->out; } } break; case NFA_MARK: case NFA_MARK_GT: case NFA_MARK_LT: { pos_T *pos = getmark_buf(rex.reg_buf, t->state->val, FALSE); // Compare the mark position to the match position, if the mark // exists and mark is set in reg_buf. if (pos != NULL && pos->lnum > 0) { colnr_T pos_col = pos->lnum == rex.lnum + rex.reg_firstlnum && pos->col == MAXCOL ? (colnr_T)STRLEN(reg_getline( pos->lnum - rex.reg_firstlnum)) : pos->col; result = (pos->lnum == rex.lnum + rex.reg_firstlnum ? (pos_col == (colnr_T)(rex.input - rex.line) ? t->state->c == NFA_MARK : (pos_col < (colnr_T)(rex.input - rex.line) ? t->state->c == NFA_MARK_GT : t->state->c == NFA_MARK_LT)) : (pos->lnum < rex.lnum + rex.reg_firstlnum ? t->state->c == NFA_MARK_GT : t->state->c == NFA_MARK_LT)); if (result) { add_here = TRUE; add_state = t->state->out; } } break; } case NFA_CURSOR: result = (rex.reg_win != NULL && (rex.lnum + rex.reg_firstlnum == rex.reg_win->w_cursor.lnum) && ((colnr_T)(rex.input - rex.line) == rex.reg_win->w_cursor.col)); if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_VISUAL: result = reg_match_visual(); if (result) { add_here = TRUE; add_state = t->state->out; } break; case NFA_MOPEN1: case NFA_MOPEN2: case NFA_MOPEN3: case NFA_MOPEN4: case NFA_MOPEN5: case NFA_MOPEN6: case NFA_MOPEN7: case NFA_MOPEN8: case NFA_MOPEN9: #ifdef FEAT_SYN_HL case NFA_ZOPEN: case NFA_ZOPEN1: case NFA_ZOPEN2: case NFA_ZOPEN3: case NFA_ZOPEN4: case NFA_ZOPEN5: case NFA_ZOPEN6: case NFA_ZOPEN7: case NFA_ZOPEN8: case NFA_ZOPEN9: #endif case NFA_NOPEN: case NFA_ZSTART: // These states are only added to be able to bail out when // they are added again, nothing is to be done. break; default: // regular character { int c = t->state->c; #ifdef DEBUG if (c < 0) siemsg("INTERNAL: Negative state char: %ld", c); #endif result = (c == curc); if (!result && rex.reg_ic) result = MB_CASEFOLD(c) == MB_CASEFOLD(curc); // If rex.reg_icombine is not set only skip over the character // itself. When it is set skip over composing characters. if (result && enc_utf8 && !rex.reg_icombine) clen = utf_ptr2len(rex.input); ADD_STATE_IF_MATCH(t->state); break; } } // switch (t->state->c) if (add_state != NULL) { nfa_pim_T *pim; nfa_pim_T pim_copy; if (t->pim.result == NFA_PIM_UNUSED) pim = NULL; else pim = &t->pim; // Handle the postponed invisible match if the match might end // without advancing and before the end of the line. if (pim != NULL && (clen == 0 || match_follows(add_state, 0))) { if (pim->result == NFA_PIM_TODO) { #ifdef ENABLE_LOG fprintf(log_fd, "\n"); fprintf(log_fd, "==================================\n"); fprintf(log_fd, "Postponed recursive nfa_regmatch()\n"); fprintf(log_fd, "\n"); #endif result = recursive_regmatch(pim->state, pim, prog, submatch, m, &listids, &listids_len); pim->result = result ? NFA_PIM_MATCH : NFA_PIM_NOMATCH; // for \@! and \@<! it is a match when the result is // FALSE if (result != (pim->state->c == NFA_START_INVISIBLE_NEG || pim->state->c == NFA_START_INVISIBLE_NEG_FIRST || pim->state->c == NFA_START_INVISIBLE_BEFORE_NEG || pim->state->c == NFA_START_INVISIBLE_BEFORE_NEG_FIRST)) { // Copy submatch info from the recursive call copy_sub_off(&pim->subs.norm, &m->norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&pim->subs.synt, &m->synt); #endif } } else { result = (pim->result == NFA_PIM_MATCH); #ifdef ENABLE_LOG fprintf(log_fd, "\n"); fprintf(log_fd, "Using previous recursive nfa_regmatch() result, result == %d\n", pim->result); fprintf(log_fd, "MATCH = %s\n", result == TRUE ? "OK" : "FALSE"); fprintf(log_fd, "\n"); #endif } // for \@! and \@<! it is a match when result is FALSE if (result != (pim->state->c == NFA_START_INVISIBLE_NEG || pim->state->c == NFA_START_INVISIBLE_NEG_FIRST || pim->state->c == NFA_START_INVISIBLE_BEFORE_NEG || pim->state->c == NFA_START_INVISIBLE_BEFORE_NEG_FIRST)) { // Copy submatch info from the recursive call copy_sub_off(&t->subs.norm, &pim->subs.norm); #ifdef FEAT_SYN_HL if (rex.nfa_has_zsubexpr) copy_sub_off(&t->subs.synt, &pim->subs.synt); #endif } else // look-behind match failed, don't add the state continue; // Postponed invisible match was handled, don't add it to // following states. pim = NULL; } // If "pim" points into l->t it will become invalid when // adding the state causes the list to be reallocated. Make a // local copy to avoid that. if (pim == &t->pim) { copy_pim(&pim_copy, pim); pim = &pim_copy; } if (add_here) r = addstate_here(thislist, add_state, &t->subs, pim, &listidx); else { r = addstate(nextlist, add_state, &t->subs, pim, add_off); if (add_count > 0) nextlist->t[nextlist->n - 1].count = add_count; } if (r == NULL) { nfa_match = NFA_TOO_EXPENSIVE; goto theend; } } } // for (thislist = thislist; thislist->state; thislist++) // Look for the start of a match in the current position by adding the // start state to the list of states. // The first found match is the leftmost one, thus the order of states // matters! // Do not add the start state in recursive calls of nfa_regmatch(), // because recursive calls should only start in the first position. // Unless "nfa_endp" is not NULL, then we match the end position. // Also don't start a match past the first line. if (nfa_match == FALSE && ((toplevel && rex.lnum == 0 && clen != 0 && (rex.reg_maxcol == 0 || (colnr_T)(rex.input - rex.line) < rex.reg_maxcol)) || (nfa_endp != NULL && (REG_MULTI ? (rex.lnum < nfa_endp->se_u.pos.lnum || (rex.lnum == nfa_endp->se_u.pos.lnum && (int)(rex.input - rex.line) < nfa_endp->se_u.pos.col)) : rex.input < nfa_endp->se_u.ptr)))) { #ifdef ENABLE_LOG fprintf(log_fd, "(---) STARTSTATE\n"); #endif // Inline optimized code for addstate() if we know the state is // the first MOPEN. if (toplevel) { int add = TRUE; int c; if (prog->regstart != NUL && clen != 0) { if (nextlist->n == 0) { colnr_T col = (colnr_T)(rex.input - rex.line) + clen; // Nextlist is empty, we can skip ahead to the // character that must appear at the start. if (skip_to_start(prog->regstart, &col) == FAIL) break; #ifdef ENABLE_LOG fprintf(log_fd, " Skipping ahead %d bytes to regstart\n", col - ((colnr_T)(rex.input - rex.line) + clen)); #endif rex.input = rex.line + col - clen; } else { // Checking if the required start character matches is // cheaper than adding a state that won't match. c = PTR2CHAR(rex.input + clen); if (c != prog->regstart && (!rex.reg_ic || MB_CASEFOLD(c) != MB_CASEFOLD(prog->regstart))) { #ifdef ENABLE_LOG fprintf(log_fd, " Skipping start state, regstart does not match\n"); #endif add = FALSE; } } } if (add) { if (REG_MULTI) m->norm.list.multi[0].start_col = (colnr_T)(rex.input - rex.line) + clen; else m->norm.list.line[0].start = rex.input + clen; if (addstate(nextlist, start->out, m, NULL, clen) == NULL) { nfa_match = NFA_TOO_EXPENSIVE; goto theend; } } } else { if (addstate(nextlist, start, m, NULL, clen) == NULL) { nfa_match = NFA_TOO_EXPENSIVE; goto theend; } } } #ifdef ENABLE_LOG fprintf(log_fd, ">>> Thislist had %d states available: ", thislist->n); { int i; for (i = 0; i < thislist->n; i++) fprintf(log_fd, "%d ", abs(thislist->t[i].state->id)); } fprintf(log_fd, "\n"); #endif nextchar: // Advance to the next character, or advance to the next line, or // finish. if (clen != 0) rex.input += clen; else if (go_to_nextline || (nfa_endp != NULL && REG_MULTI && rex.lnum < nfa_endp->se_u.pos.lnum)) reg_nextline(); else break; // Allow interrupting with CTRL-C. line_breakcheck(); if (got_int) break; #ifdef FEAT_RELTIME // Check for timeout once in a twenty times to avoid overhead. if (nfa_time_limit != NULL && ++nfa_time_count == 20) { nfa_time_count = 0; if (nfa_did_time_out()) break; } #endif } #ifdef ENABLE_LOG if (log_fd != stderr) fclose(log_fd); log_fd = NULL; #endif theend: // Free memory vim_free(list[0].t); vim_free(list[1].t); vim_free(listids); #undef ADD_STATE_IF_MATCH #ifdef NFA_REGEXP_DEBUG_LOG fclose(debug); #endif return nfa_match; } /* * Try match of "prog" with at rex.line["col"]. * Returns <= 0 for failure, number of lines contained in the match otherwise. */ static long nfa_regtry( nfa_regprog_T *prog, colnr_T col, proftime_T *tm UNUSED, // timeout limit or NULL int *timed_out UNUSED) // flag set on timeout or NULL { int i; regsubs_T subs, m; nfa_state_T *start = prog->start; int result; #ifdef ENABLE_LOG FILE *f; #endif rex.input = rex.line + col; #ifdef FEAT_RELTIME nfa_time_limit = tm; nfa_timed_out = timed_out; nfa_time_count = 0; #endif #ifdef ENABLE_LOG f = fopen(NFA_REGEXP_RUN_LOG, "a"); if (f != NULL) { fprintf(f, "\n\n\t=======================================================\n"); #ifdef DEBUG fprintf(f, "\tRegexp is \"%s\"\n", nfa_regengine.expr); #endif fprintf(f, "\tInput text is \"%s\" \n", rex.input); fprintf(f, "\t=======================================================\n\n"); nfa_print_state(f, start); fprintf(f, "\n\n"); fclose(f); } else emsg("Could not open temporary log file for writing"); #endif clear_sub(&subs.norm); clear_sub(&m.norm); #ifdef FEAT_SYN_HL clear_sub(&subs.synt); clear_sub(&m.synt); #endif result = nfa_regmatch(prog, start, &subs, &m); if (result == FALSE) return 0; else if (result == NFA_TOO_EXPENSIVE) return result; cleanup_subexpr(); if (REG_MULTI) { for (i = 0; i < subs.norm.in_use; i++) { rex.reg_startpos[i].lnum = subs.norm.list.multi[i].start_lnum; rex.reg_startpos[i].col = subs.norm.list.multi[i].start_col; rex.reg_endpos[i].lnum = subs.norm.list.multi[i].end_lnum; rex.reg_endpos[i].col = subs.norm.list.multi[i].end_col; } if (rex.reg_startpos[0].lnum < 0) { rex.reg_startpos[0].lnum = 0; rex.reg_startpos[0].col = col; } if (rex.reg_endpos[0].lnum < 0) { // pattern has a \ze but it didn't match, use current end rex.reg_endpos[0].lnum = rex.lnum; rex.reg_endpos[0].col = (int)(rex.input - rex.line); } else // Use line number of "\ze". rex.lnum = rex.reg_endpos[0].lnum; } else { for (i = 0; i < subs.norm.in_use; i++) { rex.reg_startp[i] = subs.norm.list.line[i].start; rex.reg_endp[i] = subs.norm.list.line[i].end; } if (rex.reg_startp[0] == NULL) rex.reg_startp[0] = rex.line + col; if (rex.reg_endp[0] == NULL) rex.reg_endp[0] = rex.input; } #ifdef FEAT_SYN_HL // Package any found \z(...\) matches for export. Default is none. unref_extmatch(re_extmatch_out); re_extmatch_out = NULL; if (prog->reghasz == REX_SET) { cleanup_zsubexpr(); re_extmatch_out = make_extmatch(); if (re_extmatch_out == NULL) return 0; // Loop over \z1, \z2, etc. There is no \z0. for (i = 1; i < subs.synt.in_use; i++) { if (REG_MULTI) { struct multipos *mpos = &subs.synt.list.multi[i]; // Only accept single line matches that are valid. if (mpos->start_lnum >= 0 && mpos->start_lnum == mpos->end_lnum && mpos->end_col >= mpos->start_col) re_extmatch_out->matches[i] = vim_strnsave(reg_getline(mpos->start_lnum) + mpos->start_col, mpos->end_col - mpos->start_col); } else { struct linepos *lpos = &subs.synt.list.line[i]; if (lpos->start != NULL && lpos->end != NULL) re_extmatch_out->matches[i] = vim_strnsave(lpos->start, lpos->end - lpos->start); } } } #endif return 1 + rex.lnum; } /* * Match a regexp against a string ("line" points to the string) or multiple * lines (if "line" is NULL, use reg_getline()). * * Returns <= 0 for failure, number of lines contained in the match otherwise. */ static long nfa_regexec_both( char_u *line, colnr_T startcol, // column to start looking for match proftime_T *tm, // timeout limit or NULL int *timed_out) // flag set on timeout or NULL { nfa_regprog_T *prog; long retval = 0L; int i; colnr_T col = startcol; if (REG_MULTI) { prog = (nfa_regprog_T *)rex.reg_mmatch->regprog; line = reg_getline((linenr_T)0); // relative to the cursor rex.reg_startpos = rex.reg_mmatch->startpos; rex.reg_endpos = rex.reg_mmatch->endpos; } else { prog = (nfa_regprog_T *)rex.reg_match->regprog; rex.reg_startp = rex.reg_match->startp; rex.reg_endp = rex.reg_match->endp; } // Be paranoid... if (prog == NULL || line == NULL) { iemsg(_(e_null)); goto theend; } // If pattern contains "\c" or "\C": overrule value of rex.reg_ic if (prog->regflags & RF_ICASE) rex.reg_ic = TRUE; else if (prog->regflags & RF_NOICASE) rex.reg_ic = FALSE; // If pattern contains "\Z" overrule value of rex.reg_icombine if (prog->regflags & RF_ICOMBINE) rex.reg_icombine = TRUE; rex.line = line; rex.lnum = 0; // relative to line rex.nfa_has_zend = prog->has_zend; rex.nfa_has_backref = prog->has_backref; rex.nfa_nsubexpr = prog->nsubexp; rex.nfa_listid = 1; rex.nfa_alt_listid = 2; #ifdef DEBUG nfa_regengine.expr = prog->pattern; #endif if (prog->reganch && col > 0) return 0L; rex.need_clear_subexpr = TRUE; #ifdef FEAT_SYN_HL // Clear the external match subpointers if necessary. if (prog->reghasz == REX_SET) { rex.nfa_has_zsubexpr = TRUE; rex.need_clear_zsubexpr = TRUE; } else { rex.nfa_has_zsubexpr = FALSE; rex.need_clear_zsubexpr = FALSE; } #endif if (prog->regstart != NUL) { // Skip ahead until a character we know the match must start with. // When there is none there is no match. if (skip_to_start(prog->regstart, &col) == FAIL) return 0L; // If match_text is set it contains the full text that must match. // Nothing else to try. Doesn't handle combining chars well. if (prog->match_text != NULL && !rex.reg_icombine) return find_match_text(col, prog->regstart, prog->match_text); } // If the start column is past the maximum column: no need to try. if (rex.reg_maxcol > 0 && col >= rex.reg_maxcol) goto theend; // Set the "nstate" used by nfa_regcomp() to zero to trigger an error when // it's accidentally used during execution. nstate = 0; for (i = 0; i < prog->nstate; ++i) { prog->state[i].id = i; prog->state[i].lastlist[0] = 0; prog->state[i].lastlist[1] = 0; } retval = nfa_regtry(prog, col, tm, timed_out); #ifdef DEBUG nfa_regengine.expr = NULL; #endif theend: if (retval > 0) { // Make sure the end is never before the start. Can happen when \zs and // \ze are used. if (REG_MULTI) { lpos_T *start = &rex.reg_mmatch->startpos[0]; lpos_T *end = &rex.reg_mmatch->endpos[0]; if (end->lnum < start->lnum || (end->lnum == start->lnum && end->col < start->col)) rex.reg_mmatch->endpos[0] = rex.reg_mmatch->startpos[0]; } else { if (rex.reg_match->endp[0] < rex.reg_match->startp[0]) rex.reg_match->endp[0] = rex.reg_match->startp[0]; } } return retval; } /* * Compile a regular expression into internal code for the NFA matcher. * Returns the program in allocated space. Returns NULL for an error. */ static regprog_T * nfa_regcomp(char_u *expr, int re_flags) { nfa_regprog_T *prog = NULL; size_t prog_size; int *postfix; if (expr == NULL) return NULL; #ifdef DEBUG nfa_regengine.expr = expr; #endif nfa_re_flags = re_flags; init_class_tab(); if (nfa_regcomp_start(expr, re_flags) == FAIL) return NULL; // Build postfix form of the regexp. Needed to build the NFA // (and count its size). postfix = re2post(); if (postfix == NULL) goto fail; // Cascaded (syntax?) error /* * In order to build the NFA, we parse the input regexp twice: * 1. first pass to count size (so we can allocate space) * 2. second to emit code */ #ifdef ENABLE_LOG { FILE *f = fopen(NFA_REGEXP_RUN_LOG, "a"); if (f != NULL) { fprintf(f, "\n*****************************\n\n\n\n\tCompiling regexp \"%s\"... hold on !\n", expr); fclose(f); } } #endif /* * PASS 1 * Count number of NFA states in "nstate". Do not build the NFA. */ post2nfa(postfix, post_ptr, TRUE); // allocate the regprog with space for the compiled regexp prog_size = sizeof(nfa_regprog_T) + sizeof(nfa_state_T) * (nstate - 1); prog = alloc(prog_size); if (prog == NULL) goto fail; state_ptr = prog->state; prog->re_in_use = FALSE; /* * PASS 2 * Build the NFA */ prog->start = post2nfa(postfix, post_ptr, FALSE); if (prog->start == NULL) goto fail; prog->regflags = regflags; prog->engine = &nfa_regengine; prog->nstate = nstate; prog->has_zend = rex.nfa_has_zend; prog->has_backref = rex.nfa_has_backref; prog->nsubexp = regnpar; nfa_postprocess(prog); prog->reganch = nfa_get_reganch(prog->start, 0); prog->regstart = nfa_get_regstart(prog->start, 0); prog->match_text = nfa_get_match_text(prog->start); #ifdef ENABLE_LOG nfa_postfix_dump(expr, OK); nfa_dump(prog); #endif #ifdef FEAT_SYN_HL // Remember whether this pattern has any \z specials in it. prog->reghasz = re_has_z; #endif prog->pattern = vim_strsave(expr); #ifdef DEBUG nfa_regengine.expr = NULL; #endif out: VIM_CLEAR(post_start); post_ptr = post_end = NULL; state_ptr = NULL; return (regprog_T *)prog; fail: VIM_CLEAR(prog); #ifdef ENABLE_LOG nfa_postfix_dump(expr, FAIL); #endif #ifdef DEBUG nfa_regengine.expr = NULL; #endif goto out; } /* * Free a compiled regexp program, returned by nfa_regcomp(). */ static void nfa_regfree(regprog_T *prog) { if (prog != NULL) { vim_free(((nfa_regprog_T *)prog)->match_text); vim_free(((nfa_regprog_T *)prog)->pattern); vim_free(prog); } } /* * Match a regexp against a string. * "rmp->regprog" is a compiled regexp as returned by nfa_regcomp(). * Uses curbuf for line count and 'iskeyword'. * If "line_lbr" is TRUE consider a "\n" in "line" to be a line break. * * Returns <= 0 for failure, number of lines contained in the match otherwise. */ static int nfa_regexec_nl( regmatch_T *rmp, char_u *line, // string to match against colnr_T col, // column to start looking for match int line_lbr) { rex.reg_match = rmp; rex.reg_mmatch = NULL; rex.reg_maxline = 0; rex.reg_line_lbr = line_lbr; rex.reg_buf = curbuf; rex.reg_win = NULL; rex.reg_ic = rmp->rm_ic; rex.reg_icombine = FALSE; rex.reg_maxcol = 0; return nfa_regexec_both(line, col, NULL, NULL); } /* * Match a regexp against multiple lines. * "rmp->regprog" is a compiled regexp as returned by vim_regcomp(). * Uses curbuf for line count and 'iskeyword'. * * Return <= 0 if there is no match. Return number of lines contained in the * match otherwise. * * Note: the body is the same as bt_regexec() except for nfa_regexec_both() * * ! Also NOTE : match may actually be in another line. e.g.: * when r.e. is \nc, cursor is at 'a' and the text buffer looks like * * +-------------------------+ * |a | * |b | * |c | * | | * +-------------------------+ * * then nfa_regexec_multi() returns 3. while the original * vim_regexec_multi() returns 0 and a second call at line 2 will return 2. * * FIXME if this behavior is not compatible. */ static long nfa_regexec_multi( regmmatch_T *rmp, win_T *win, // window in which to search or NULL buf_T *buf, // buffer in which to search linenr_T lnum, // nr of line to start looking for match colnr_T col, // column to start looking for match proftime_T *tm, // timeout limit or NULL int *timed_out) // flag set on timeout or NULL { init_regexec_multi(rmp, win, buf, lnum); return nfa_regexec_both(NULL, col, tm, timed_out); } #ifdef DEBUG # undef ENABLE_LOG #endif