Mercurial > vim
annotate runtime/tools/unicode.vim @ 25062:042560a16d4e v8.2.3068
patch 8.2.3068: Unicode tables are slightly outdated
Commit: https://github.com/vim/vim/commit/d887297ad0164516dd52cdab0308c3626337d124
Author: Christian Brabandt <cb@256bit.org>
Date: Sun Jun 27 21:30:14 2021 +0200
patch 8.2.3068: Unicode tables are slightly outdated
Problem: Unicode tables are slightly outdated.
Solution: Update the tables for Unicode release 13. (Christian Brabandt
closes #8430)
| author | Bram Moolenaar <Bram@vim.org> |
|---|---|
| date | Sun, 27 Jun 2021 21:45:02 +0200 |
| parents | bbca88cd13d5 |
| children | 3ee335235412 |
| rev | line source |
|---|---|
| 2041 | 1 " Script to extract tables from Unicode .txt files, to be used in src/mbyte.c. |
| 2 " The format of the UnicodeData.txt file is explained here: | |
| 3 " http://www.unicode.org/Public/5.1.0/ucd/UCD.html | |
| 4 " For the other files see the header. | |
| 5 " | |
|
11539
f7ba69508fd5
patch 8.0.0652: unicode information is outdated
Christian Brabandt <cb@256bit.org>
parents:
8706
diff
changeset
|
6 " Might need to update the URL to the emoji-data.txt |
| 2041 | 7 " Usage: Vim -S <this-file> |
| 8 " | |
| 9 " Author: Bram Moolenaar | |
| 21991 | 10 " Last Update: 2020 Aug 24 |
| 2041 | 11 |
| 12 " Parse lines of UnicodeData.txt. Creates a list of lists in s:dataprops. | |
| 13 func! ParseDataToProps() | |
| 14 let s:dataprops = [] | |
| 15 let lnum = 1 | |
| 16 while lnum <= line('$') | |
| 17 let l = split(getline(lnum), '\s*;\s*', 1) | |
| 18 if len(l) != 15 | |
| 19 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 15' | |
| 20 return | |
| 21 endif | |
| 22 call add(s:dataprops, l) | |
| 23 let lnum += 1 | |
| 24 endwhile | |
| 25 endfunc | |
| 26 | |
| 27 " Parse lines of CaseFolding.txt. Creates a list of lists in s:foldprops. | |
| 28 func! ParseFoldProps() | |
| 29 let s:foldprops = [] | |
| 30 let lnum = 1 | |
| 31 while lnum <= line('$') | |
| 32 let line = getline(lnum) | |
| 33 if line !~ '^#' && line !~ '^\s*$' | |
| 34 let l = split(line, '\s*;\s*', 1) | |
| 35 if len(l) != 4 | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
36 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 4' |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
37 return |
| 2041 | 38 endif |
| 39 call add(s:foldprops, l) | |
| 40 endif | |
| 41 let lnum += 1 | |
| 42 endwhile | |
| 43 endfunc | |
| 44 | |
| 45 " Parse lines of EastAsianWidth.txt. Creates a list of lists in s:widthprops. | |
| 46 func! ParseWidthProps() | |
| 47 let s:widthprops = [] | |
| 48 let lnum = 1 | |
| 49 while lnum <= line('$') | |
| 50 let line = getline(lnum) | |
| 51 if line !~ '^#' && line !~ '^\s*$' | |
| 52 let l = split(line, '\s*;\s*', 1) | |
| 53 if len(l) != 2 | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
54 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 2' |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
55 return |
| 2041 | 56 endif |
| 57 call add(s:widthprops, l) | |
| 58 endif | |
| 59 let lnum += 1 | |
| 60 endwhile | |
| 61 endfunc | |
| 62 | |
| 63 " Build the toLower or toUpper table in a new buffer. | |
| 64 " Uses s:dataprops. | |
| 65 func! BuildCaseTable(name, index) | |
| 66 let start = -1 | |
| 67 let end = -1 | |
| 68 let step = 0 | |
| 69 let add = -1 | |
| 70 let ranges = [] | |
| 71 for p in s:dataprops | |
| 72 if p[a:index] != '' | |
| 73 let n = ('0x' . p[0]) + 0 | |
| 74 let nl = ('0x' . p[a:index]) + 0 | |
| 75 if start >= 0 && add == nl - n && (step == 0 || n - end == step) | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
76 " continue with same range. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
77 let step = n - end |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
78 let end = n |
| 2041 | 79 else |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
80 if start >= 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
81 " produce previous range |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
82 call Range(ranges, start, end, step, add) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
83 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
84 let start = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
85 let end = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
86 let step = 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
87 let add = nl - n |
| 2041 | 88 endif |
| 89 endif | |
| 90 endfor | |
| 91 if start >= 0 | |
| 92 call Range(ranges, start, end, step, add) | |
| 93 endif | |
| 94 | |
| 95 " New buffer to put the result in. | |
| 96 new | |
| 97 exe "file to" . a:name | |
| 98 call setline(1, "static convertStruct to" . a:name . "[] =") | |
| 99 call setline(2, "{") | |
| 100 call append('$', ranges) | |
| 101 call setline('$', getline('$')[:-2]) " remove last comma | |
| 102 call setline(line('$') + 1, "};") | |
| 103 wincmd p | |
| 104 endfunc | |
| 105 | |
| 106 " Build the foldCase table in a new buffer. | |
| 107 " Uses s:foldprops. | |
| 108 func! BuildFoldTable() | |
| 109 let start = -1 | |
| 110 let end = -1 | |
| 111 let step = 0 | |
| 112 let add = -1 | |
| 113 let ranges = [] | |
| 114 for p in s:foldprops | |
| 115 if p[1] == 'C' || p[1] == 'S' | |
| 116 let n = ('0x' . p[0]) + 0 | |
| 117 let nl = ('0x' . p[2]) + 0 | |
| 118 if start >= 0 && add == nl - n && (step == 0 || n - end == step) | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
119 " continue with same range. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
120 let step = n - end |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
121 let end = n |
| 2041 | 122 else |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
123 if start >= 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
124 " produce previous range |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
125 call Range(ranges, start, end, step, add) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
126 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
127 let start = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
128 let end = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
129 let step = 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
130 let add = nl - n |
| 2041 | 131 endif |
| 132 endif | |
| 133 endfor | |
| 134 if start >= 0 | |
| 135 call Range(ranges, start, end, step, add) | |
| 136 endif | |
| 137 | |
| 138 " New buffer to put the result in. | |
| 139 new | |
| 140 file foldCase | |
| 141 call setline(1, "static convertStruct foldCase[] =") | |
| 142 call setline(2, "{") | |
| 143 call append('$', ranges) | |
| 144 call setline('$', getline('$')[:-2]) " remove last comma | |
| 145 call setline(line('$') + 1, "};") | |
| 146 wincmd p | |
| 147 endfunc | |
| 148 | |
| 149 func! Range(ranges, start, end, step, add) | |
| 150 let s = printf("\t{0x%x,0x%x,%d,%d},", a:start, a:end, a:step == 0 ? -1 : a:step, a:add) | |
| 151 call add(a:ranges, s) | |
| 152 endfunc | |
| 153 | |
| 154 " Build the combining table. | |
| 155 " Uses s:dataprops. | |
| 156 func! BuildCombiningTable() | |
| 157 let start = -1 | |
| 158 let end = -1 | |
| 159 let ranges = [] | |
| 160 for p in s:dataprops | |
| 161 if p[2] == 'Mn' || p[2] == 'Mc' || p[2] == 'Me' | |
| 162 let n = ('0x' . p[0]) + 0 | |
| 163 if start >= 0 && end + 1 == n | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
164 " continue with same range. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
165 let end = n |
| 2041 | 166 else |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
167 if start >= 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
168 " produce previous range |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
169 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end)) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
170 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
171 let start = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
172 let end = n |
| 2041 | 173 endif |
| 174 endif | |
| 175 endfor | |
| 176 if start >= 0 | |
| 177 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end)) | |
| 178 endif | |
| 179 | |
| 180 " New buffer to put the result in. | |
| 181 new | |
| 182 file combining | |
| 183 call setline(1, " static struct interval combining[] =") | |
| 184 call setline(2, " {") | |
| 185 call append('$', ranges) | |
| 186 call setline('$', getline('$')[:-2]) " remove last comma | |
| 187 call setline(line('$') + 1, " };") | |
| 188 wincmd p | |
| 189 endfunc | |
| 190 | |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
191 " Build the double width or ambiguous width table in a new buffer. |
| 2041 | 192 " Uses s:widthprops and s:dataprops. |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
193 func! BuildWidthTable(pattern, tableName) |
| 2041 | 194 let start = -1 |
| 195 let end = -1 | |
| 196 let ranges = [] | |
| 197 let dataidx = 0 | |
|
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
198 " Account for indentation differences between ambiguous and doublewidth |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
199 " table in mbyte.c |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
200 if a:pattern == 'A' |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
201 let spc = ' ' |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
202 else |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
203 let spc = "\t" |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
204 endif |
| 2041 | 205 for p in s:widthprops |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
206 if p[1][0] =~ a:pattern |
|
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
207 if p[0] =~ '\.\.' |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
208 " It is a range. we don't check for composing char then. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
209 let rng = split(p[0], '\.\.') |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
210 if len(rng) != 2 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
211 echoerr "Cannot parse range: '" . p[0] . "' in width table" |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
212 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
213 let n = ('0x' . rng[0]) + 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
214 let n_last = ('0x' . rng[1]) + 0 |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
215 else |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
216 let n = ('0x' . p[0]) + 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
217 let n_last = n |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
218 endif |
| 2041 | 219 " Find this char in the data table. |
| 220 while 1 | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
221 let dn = ('0x' . s:dataprops[dataidx][0]) + 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
222 if dn >= n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
223 break |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
224 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
225 let dataidx += 1 |
| 2041 | 226 endwhile |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
227 if dn != n && n_last == n |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
228 echoerr "Cannot find character " . n . " in data table" |
| 2041 | 229 endif |
| 230 " Only use the char when it's not a composing char. | |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
231 " But use all chars from a range. |
| 2041 | 232 let dp = s:dataprops[dataidx] |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
233 if n_last > n || (dp[2] != 'Mn' && dp[2] != 'Mc' && dp[2] != 'Me') |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
234 if start >= 0 && end + 1 == n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
235 " continue with same range. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
236 else |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
237 if start >= 0 |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
238 " produce previous range |
|
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
239 call add(ranges, printf("%s{0x%04x, 0x%04x},", spc, start, end)) |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
240 if a:pattern == 'A' |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
241 call add(s:ambitable, [start, end]) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
242 else |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
243 call add(s:doubletable, [start, end]) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
244 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
245 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
246 let start = n |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
247 endif |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
248 let end = n_last |
| 2041 | 249 endif |
| 250 endif | |
| 251 endfor | |
| 252 if start >= 0 | |
|
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
253 call add(ranges, printf("%s{0x%04x, 0x%04x},", spc, start, end)) |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
254 if a:pattern == 'A' |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
255 call add(s:ambitable, [start, end]) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
256 else |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
257 call add(s:doubletable, [start, end]) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
258 endif |
| 2041 | 259 endif |
| 260 | |
| 261 " New buffer to put the result in. | |
| 262 new | |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
263 exe "file " . a:tableName |
|
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
264 if a:pattern == 'A' |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
265 call setline(1, "static struct interval " . a:tableName . "[] =") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
266 call setline(2, "{") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
267 else |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
268 call setline(1, " static struct interval " . a:tableName . "[] =") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
269 call setline(2, " {") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
270 endif |
| 2041 | 271 call append('$', ranges) |
| 272 call setline('$', getline('$')[:-2]) " remove last comma | |
|
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
273 if a:pattern == 'A' |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
274 call setline(line('$') + 1, "};") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
275 else |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
276 call setline(line('$') + 1, " };") |
|
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
277 endif |
| 2041 | 278 wincmd p |
| 279 endfunc | |
| 280 | |
| 21991 | 281 |
| 282 " Get characters from a list of lines in form "12ab .." or "12ab..56cd ..." | |
| 283 " and put them in dictionary "chardict" | |
| 284 func AddLinesToCharDict(lines, chardict) | |
| 285 for line in a:lines | |
| 286 let tokens = split(line, '\.\.') | |
| 287 let first = str2nr(tokens[0], 16) | |
| 288 if len(tokens) == 1 | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
289 let last = first |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
290 else |
| 21991 | 291 let last = str2nr(tokens[1], 16) |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
292 endif |
| 21991 | 293 for nr in range(first, last) |
| 294 let a:chardict[nr] = 1 | |
| 295 endfor | |
| 296 endfor | |
| 297 endfunc | |
|
8706
f63892cfe283
commit https://github.com/vim/vim/commit/6a08454b93784c92296d4c08456401cbaa74c9d5
Christian Brabandt <cb@256bit.org>
parents:
8680
diff
changeset
|
298 |
| 21991 | 299 func Test_AddLinesToCharDict() |
| 300 let dict = {} | |
| 301 call AddLinesToCharDict([ | |
| 302 \ '1234 blah blah', | |
| 303 \ '1235 blah blah', | |
| 304 \ '12a0..12a2 blah blah', | |
| 305 \ '12a1 blah blah', | |
| 306 \ ], dict) | |
| 307 call assert_equal({0x1234: 1, 0x1235: 1, | |
| 308 \ 0x12a0: 1, 0x12a1: 1, 0x12a2: 1, | |
| 309 \ }, dict) | |
| 310 if v:errors != [] | |
| 311 echoerr 'AddLinesToCharDict' v:errors | |
| 312 return 1 | |
| 313 endif | |
| 314 return 0 | |
| 315 endfunc | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
316 |
| 21991 | 317 |
| 318 func CharDictToPairList(chardict) | |
| 319 let result = [] | |
| 320 let keys = keys(a:chardict)->map('str2nr(v:val)')->sort('N') | |
| 321 let low = keys[0] | |
| 322 let high = keys[0] | |
| 323 for key in keys | |
| 324 if key > high + 1 | |
| 325 call add(result, [low, high]) | |
| 326 let low = key | |
| 327 let high = key | |
| 328 else | |
| 329 let high = key | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
330 endif |
|
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
331 endfor |
| 21991 | 332 call add(result, [low, high]) |
| 333 return result | |
| 334 endfunc | |
| 335 | |
| 336 func Test_CharDictToPairList() | |
| 337 let dict = {0x1020: 1, 0x1021: 1, 0x1022: 1, | |
| 338 \ 0x1024: 1, | |
| 339 \ 0x2022: 1, | |
| 340 \ 0x2024: 1, 0x2025: 1} | |
| 341 call assert_equal([ | |
| 342 \ [0x1020, 0x1022], | |
| 343 \ [0x1024, 0x1024], | |
| 344 \ [0x2022, 0x2022], | |
| 345 \ [0x2024, 0x2025], | |
| 346 \ ], CharDictToPairList(dict)) | |
| 347 if v:errors != [] | |
| 348 echoerr 'CharDictToPairList' v:errors | |
| 349 return 1 | |
| 350 endif | |
| 351 return 0 | |
| 352 endfunc | |
| 353 | |
| 354 | |
| 355 " Build the amoji width table in a new buffer. | |
| 356 func BuildEmojiTable() | |
| 357 " First make the table for all emojis. | |
| 358 let pattern = '; Emoji\s\+#\s' | |
| 359 let lines = map(filter(filter(getline(1, '$'), 'v:val=~"^[1-9]"'), 'v:val=~pattern'), 'matchstr(v:val,"^\\S\\+")') | |
| 360 | |
| 361 " Make a dictionary with an entry for each character. | |
| 362 let chardict = {} | |
| 363 call AddLinesToCharDict(lines, chardict) | |
| 364 let pairlist = CharDictToPairList(chardict) | |
| 365 let allranges = map(pairlist, 'printf(" {0x%04x, 0x%04x},", v:val[0], v:val[1])') | |
|
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
366 |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
367 " New buffer to put the result in. |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
368 new |
| 21991 | 369 exe 'file emoji_all' |
| 370 call setline(1, "static struct interval emoji_all[] =") | |
| 371 call setline(2, "{") | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
372 call append('$', allranges) |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
373 call setline('$', getline('$')[:-2]) " remove last comma |
| 21991 | 374 call setline(line('$') + 1, "};") |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
375 wincmd p |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
376 |
| 21991 | 377 " Make the table for wide emojis. |
| 378 let pattern = '; Emoji_\(Presentation\|Modifier_Base\)\s\+#\s' | |
| 379 let lines = map(filter(filter(getline(1, '$'), 'v:val=~"^[1-9]"'), 'v:val=~pattern'), 'matchstr(v:val,"^\\S\\+")') | |
| 380 | |
| 381 " Make a dictionary with an entry for each character. | |
| 382 let chardict = {} | |
| 383 call AddLinesToCharDict(lines, chardict) | |
| 384 | |
| 385 " exclude characters that are in the "ambiguous" or "doublewidth" table | |
| 386 for ambi in s:ambitable | |
| 387 for nr in range(ambi[0], ambi[1]) | |
| 388 if has_key(chardict, nr) | |
| 389 call remove(chardict, nr) | |
| 390 endif | |
| 391 endfor | |
| 392 endfor | |
| 393 | |
| 394 for wide in s:doubletable | |
| 395 for nr in range(wide[0], wide[1]) | |
| 396 if has_key(chardict, nr) | |
| 397 call remove(chardict, nr) | |
| 398 endif | |
| 399 endfor | |
| 400 endfor | |
| 401 | |
| 402 let pairlist = CharDictToPairList(chardict) | |
| 403 let wide_ranges = map(pairlist, 'printf("\t{0x%04x, 0x%04x},", v:val[0], v:val[1])') | |
| 404 | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
405 " New buffer to put the result in. |
|
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
406 new |
| 21991 | 407 exe 'file emoji_wide' |
| 408 call setline(1, " static struct interval emoji_wide[] =") | |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
409 call setline(2, " {") |
| 21991 | 410 call append('$', wide_ranges) |
|
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
411 call setline('$', getline('$')[:-2]) " remove last comma |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
412 call setline(line('$') + 1, " };") |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
413 wincmd p |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
414 endfunc |
| 2041 | 415 |
| 21991 | 416 " First test a few things |
| 417 let v:errors = [] | |
| 418 if Test_AddLinesToCharDict() || Test_CharDictToPairList() | |
| 419 finish | |
| 420 endif | |
| 421 | |
| 422 | |
| 6864 | 423 " Try to avoid hitting E36 |
| 424 set equalalways | |
| 2041 | 425 |
| 426 " Edit the Unicode text file. Requires the netrw plugin. | |
| 427 edit http://unicode.org/Public/UNIDATA/UnicodeData.txt | |
| 428 | |
| 429 " Parse each line, create a list of lists. | |
| 430 call ParseDataToProps() | |
| 431 | |
| 432 " Build the toLower table. | |
| 433 call BuildCaseTable("Lower", 13) | |
| 434 | |
| 435 " Build the toUpper table. | |
| 436 call BuildCaseTable("Upper", 12) | |
| 437 | |
| 438 " Build the ranges of composing chars. | |
| 439 call BuildCombiningTable() | |
| 440 | |
| 441 " Edit the case folding text file. Requires the netrw plugin. | |
| 442 edit http://www.unicode.org/Public/UNIDATA/CaseFolding.txt | |
| 443 | |
| 444 " Parse each line, create a list of lists. | |
| 445 call ParseFoldProps() | |
| 446 | |
| 447 " Build the foldCase table. | |
| 448 call BuildFoldTable() | |
| 449 | |
| 450 " Edit the width text file. Requires the netrw plugin. | |
| 451 edit http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt | |
| 452 | |
| 453 " Parse each line, create a list of lists. | |
| 454 call ParseWidthProps() | |
| 455 | |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
456 " Build the double width table. |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
457 let s:doubletable = [] |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
458 call BuildWidthTable('[WF]', 'doublewidth') |
|
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
459 |
|
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
460 " Build the ambiguous width table. |
|
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
461 let s:ambitable = [] |
|
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
462 call BuildWidthTable('A', 'ambiguous') |
|
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
463 |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
464 " Edit the emoji text file. Requires the netrw plugin. |
| 21991 | 465 edit https://unicode.org/Public/emoji/12.1/emoji-data.txt |
|
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
466 |
|
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
467 " Build the emoji table. Ver. 1.0 - 6.0 |
| 21991 | 468 " Must come after the "ambiguous" and "doublewidth" tables |
| 469 call BuildEmojiTable() |