Mercurial > vim
annotate runtime/tools/unicode.vim @ 31716:9dd5bc4f2783 v9.0.1190
patch 9.0.1190: AppVeyor runs much slower with MSVC 2022
Commit: https://github.com/vim/vim/commit/60908c492237edcfe6cde995171240c9bace05b6
Author: Christopher Plewright <chris@createng.com>
Date: Fri Jan 13 15:28:14 2023 +0000
patch 9.0.1190: AppVeyor runs much slower with MSVC 2022
Problem: AppVeyor runs much slower with MSVC 2022.
Solution: Go back to MSVC 2015. (Christopher Plewright, closes https://github.com/vim/vim/issues/11810)
author | Bram Moolenaar <Bram@vim.org> |
---|---|
date | Fri, 13 Jan 2023 16:30:03 +0100 |
parents | 65c35212c772 |
children | 79113791e209 |
rev | line source |
---|---|
2041 | 1 " Script to extract tables from Unicode .txt files, to be used in src/mbyte.c. |
2 " The format of the UnicodeData.txt file is explained here: | |
3 " http://www.unicode.org/Public/5.1.0/ucd/UCD.html | |
4 " For the other files see the header. | |
5 " | |
11539
f7ba69508fd5
patch 8.0.0652: unicode information is outdated
Christian Brabandt <cb@256bit.org>
parents:
8706
diff
changeset
|
6 " Might need to update the URL to the emoji-data.txt |
2041 | 7 " Usage: Vim -S <this-file> |
8 " | |
9 " Author: Bram Moolenaar | |
21991 | 10 " Last Update: 2020 Aug 24 |
2041 | 11 |
12 " Parse lines of UnicodeData.txt. Creates a list of lists in s:dataprops. | |
13 func! ParseDataToProps() | |
14 let s:dataprops = [] | |
15 let lnum = 1 | |
16 while lnum <= line('$') | |
17 let l = split(getline(lnum), '\s*;\s*', 1) | |
18 if len(l) != 15 | |
19 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 15' | |
20 return | |
21 endif | |
22 call add(s:dataprops, l) | |
23 let lnum += 1 | |
24 endwhile | |
25 endfunc | |
26 | |
27 " Parse lines of CaseFolding.txt. Creates a list of lists in s:foldprops. | |
28 func! ParseFoldProps() | |
29 let s:foldprops = [] | |
30 let lnum = 1 | |
31 while lnum <= line('$') | |
32 let line = getline(lnum) | |
33 if line !~ '^#' && line !~ '^\s*$' | |
34 let l = split(line, '\s*;\s*', 1) | |
35 if len(l) != 4 | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
36 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 4' |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
37 return |
2041 | 38 endif |
39 call add(s:foldprops, l) | |
40 endif | |
41 let lnum += 1 | |
42 endwhile | |
43 endfunc | |
44 | |
45 " Parse lines of EastAsianWidth.txt. Creates a list of lists in s:widthprops. | |
46 func! ParseWidthProps() | |
47 let s:widthprops = [] | |
48 let lnum = 1 | |
49 while lnum <= line('$') | |
50 let line = getline(lnum) | |
51 if line !~ '^#' && line !~ '^\s*$' | |
52 let l = split(line, '\s*;\s*', 1) | |
53 if len(l) != 2 | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
54 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 2' |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
55 return |
2041 | 56 endif |
57 call add(s:widthprops, l) | |
58 endif | |
59 let lnum += 1 | |
60 endwhile | |
61 endfunc | |
62 | |
63 " Build the toLower or toUpper table in a new buffer. | |
64 " Uses s:dataprops. | |
65 func! BuildCaseTable(name, index) | |
66 let start = -1 | |
67 let end = -1 | |
68 let step = 0 | |
69 let add = -1 | |
70 let ranges = [] | |
71 for p in s:dataprops | |
72 if p[a:index] != '' | |
73 let n = ('0x' . p[0]) + 0 | |
74 let nl = ('0x' . p[a:index]) + 0 | |
75 if start >= 0 && add == nl - n && (step == 0 || n - end == step) | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
76 " continue with same range. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
77 let step = n - end |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
78 let end = n |
2041 | 79 else |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
80 if start >= 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
81 " produce previous range |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
82 call Range(ranges, start, end, step, add) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
83 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
84 let start = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
85 let end = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
86 let step = 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
87 let add = nl - n |
2041 | 88 endif |
89 endif | |
90 endfor | |
91 if start >= 0 | |
92 call Range(ranges, start, end, step, add) | |
93 endif | |
94 | |
95 " New buffer to put the result in. | |
96 new | |
97 exe "file to" . a:name | |
98 call setline(1, "static convertStruct to" . a:name . "[] =") | |
99 call setline(2, "{") | |
100 call append('$', ranges) | |
101 call setline('$', getline('$')[:-2]) " remove last comma | |
102 call setline(line('$') + 1, "};") | |
103 wincmd p | |
104 endfunc | |
105 | |
106 " Build the foldCase table in a new buffer. | |
107 " Uses s:foldprops. | |
108 func! BuildFoldTable() | |
109 let start = -1 | |
110 let end = -1 | |
111 let step = 0 | |
112 let add = -1 | |
113 let ranges = [] | |
114 for p in s:foldprops | |
115 if p[1] == 'C' || p[1] == 'S' | |
116 let n = ('0x' . p[0]) + 0 | |
117 let nl = ('0x' . p[2]) + 0 | |
118 if start >= 0 && add == nl - n && (step == 0 || n - end == step) | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
119 " continue with same range. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
120 let step = n - end |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
121 let end = n |
2041 | 122 else |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
123 if start >= 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
124 " produce previous range |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
125 call Range(ranges, start, end, step, add) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
126 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
127 let start = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
128 let end = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
129 let step = 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
130 let add = nl - n |
2041 | 131 endif |
132 endif | |
133 endfor | |
134 if start >= 0 | |
135 call Range(ranges, start, end, step, add) | |
136 endif | |
137 | |
138 " New buffer to put the result in. | |
139 new | |
140 file foldCase | |
141 call setline(1, "static convertStruct foldCase[] =") | |
142 call setline(2, "{") | |
143 call append('$', ranges) | |
144 call setline('$', getline('$')[:-2]) " remove last comma | |
145 call setline(line('$') + 1, "};") | |
146 wincmd p | |
147 endfunc | |
148 | |
149 func! Range(ranges, start, end, step, add) | |
150 let s = printf("\t{0x%x,0x%x,%d,%d},", a:start, a:end, a:step == 0 ? -1 : a:step, a:add) | |
151 call add(a:ranges, s) | |
152 endfunc | |
153 | |
154 " Build the combining table. | |
155 " Uses s:dataprops. | |
156 func! BuildCombiningTable() | |
157 let start = -1 | |
158 let end = -1 | |
159 let ranges = [] | |
160 for p in s:dataprops | |
30663
65c35212c772
patch 9.0.0666: spacing-combining characters handled as composing
Bram Moolenaar <Bram@vim.org>
parents:
30503
diff
changeset
|
161 " The 'Mc' property was removed, it does take up space. |
65c35212c772
patch 9.0.0666: spacing-combining characters handled as composing
Bram Moolenaar <Bram@vim.org>
parents:
30503
diff
changeset
|
162 if p[2] == 'Mn' || p[2] == 'Me' |
2041 | 163 let n = ('0x' . p[0]) + 0 |
164 if start >= 0 && end + 1 == n | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
165 " continue with same range. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
166 let end = n |
2041 | 167 else |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
168 if start >= 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
169 " produce previous range |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
170 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end)) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
171 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
172 let start = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
173 let end = n |
2041 | 174 endif |
175 endif | |
176 endfor | |
177 if start >= 0 | |
178 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end)) | |
179 endif | |
180 | |
181 " New buffer to put the result in. | |
182 new | |
183 file combining | |
184 call setline(1, " static struct interval combining[] =") | |
185 call setline(2, " {") | |
186 call append('$', ranges) | |
187 call setline('$', getline('$')[:-2]) " remove last comma | |
188 call setline(line('$') + 1, " };") | |
189 wincmd p | |
190 endfunc | |
191 | |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
192 " Build the double width or ambiguous width table in a new buffer. |
2041 | 193 " Uses s:widthprops and s:dataprops. |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
194 func! BuildWidthTable(pattern, tableName) |
2041 | 195 let start = -1 |
196 let end = -1 | |
197 let ranges = [] | |
198 let dataidx = 0 | |
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
199 " Account for indentation differences between ambiguous and doublewidth |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
200 " table in mbyte.c |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
201 if a:pattern == 'A' |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
202 let spc = ' ' |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
203 else |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
204 let spc = "\t" |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
205 endif |
2041 | 206 for p in s:widthprops |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
207 if p[1][0] =~ a:pattern |
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
208 if p[0] =~ '\.\.' |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
209 " It is a range. we don't check for composing char then. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
210 let rng = split(p[0], '\.\.') |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
211 if len(rng) != 2 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
212 echoerr "Cannot parse range: '" . p[0] . "' in width table" |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
213 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
214 let n = ('0x' . rng[0]) + 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
215 let n_last = ('0x' . rng[1]) + 0 |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
216 else |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
217 let n = ('0x' . p[0]) + 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
218 let n_last = n |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
219 endif |
2041 | 220 " Find this char in the data table. |
221 while 1 | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
222 let dn = ('0x' . s:dataprops[dataidx][0]) + 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
223 if dn >= n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
224 break |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
225 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
226 let dataidx += 1 |
2041 | 227 endwhile |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
228 if dn != n && n_last == n |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
229 echoerr "Cannot find character " . n . " in data table" |
2041 | 230 endif |
231 " Only use the char when it's not a composing char. | |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
232 " But use all chars from a range. |
2041 | 233 let dp = s:dataprops[dataidx] |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
234 if n_last > n || (dp[2] != 'Mn' && dp[2] != 'Mc' && dp[2] != 'Me') |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
235 if start >= 0 && end + 1 == n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
236 " continue with same range. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
237 else |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
238 if start >= 0 |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
239 " produce previous range |
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
240 call add(ranges, printf("%s{0x%04x, 0x%04x},", spc, start, end)) |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
241 if a:pattern == 'A' |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
242 call add(s:ambitable, [start, end]) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
243 else |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
244 call add(s:doubletable, [start, end]) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
245 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
246 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
247 let start = n |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
248 endif |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
249 let end = n_last |
2041 | 250 endif |
251 endif | |
252 endfor | |
253 if start >= 0 | |
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
254 call add(ranges, printf("%s{0x%04x, 0x%04x},", spc, start, end)) |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
255 if a:pattern == 'A' |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
256 call add(s:ambitable, [start, end]) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
257 else |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
258 call add(s:doubletable, [start, end]) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
259 endif |
2041 | 260 endif |
261 | |
262 " New buffer to put the result in. | |
263 new | |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
264 exe "file " . a:tableName |
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
265 if a:pattern == 'A' |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
266 call setline(1, "static struct interval " . a:tableName . "[] =") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
267 call setline(2, "{") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
268 else |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
269 call setline(1, " static struct interval " . a:tableName . "[] =") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
270 call setline(2, " {") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
271 endif |
2041 | 272 call append('$', ranges) |
273 call setline('$', getline('$')[:-2]) " remove last comma | |
25062
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
274 if a:pattern == 'A' |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
275 call setline(line('$') + 1, "};") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
276 else |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
277 call setline(line('$') + 1, " };") |
042560a16d4e
patch 8.2.3068: Unicode tables are slightly outdated
Bram Moolenaar <Bram@vim.org>
parents:
21991
diff
changeset
|
278 endif |
2041 | 279 wincmd p |
280 endfunc | |
281 | |
21991 | 282 |
283 " Get characters from a list of lines in form "12ab .." or "12ab..56cd ..." | |
284 " and put them in dictionary "chardict" | |
285 func AddLinesToCharDict(lines, chardict) | |
286 for line in a:lines | |
287 let tokens = split(line, '\.\.') | |
288 let first = str2nr(tokens[0], 16) | |
289 if len(tokens) == 1 | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
290 let last = first |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
291 else |
21991 | 292 let last = str2nr(tokens[1], 16) |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
293 endif |
21991 | 294 for nr in range(first, last) |
295 let a:chardict[nr] = 1 | |
296 endfor | |
297 endfor | |
298 endfunc | |
8706
f63892cfe283
commit https://github.com/vim/vim/commit/6a08454b93784c92296d4c08456401cbaa74c9d5
Christian Brabandt <cb@256bit.org>
parents:
8680
diff
changeset
|
299 |
21991 | 300 func Test_AddLinesToCharDict() |
301 let dict = {} | |
302 call AddLinesToCharDict([ | |
303 \ '1234 blah blah', | |
304 \ '1235 blah blah', | |
305 \ '12a0..12a2 blah blah', | |
306 \ '12a1 blah blah', | |
307 \ ], dict) | |
308 call assert_equal({0x1234: 1, 0x1235: 1, | |
309 \ 0x12a0: 1, 0x12a1: 1, 0x12a2: 1, | |
310 \ }, dict) | |
311 if v:errors != [] | |
312 echoerr 'AddLinesToCharDict' v:errors | |
313 return 1 | |
314 endif | |
315 return 0 | |
316 endfunc | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
317 |
21991 | 318 |
319 func CharDictToPairList(chardict) | |
320 let result = [] | |
321 let keys = keys(a:chardict)->map('str2nr(v:val)')->sort('N') | |
322 let low = keys[0] | |
323 let high = keys[0] | |
324 for key in keys | |
325 if key > high + 1 | |
326 call add(result, [low, high]) | |
327 let low = key | |
328 let high = key | |
329 else | |
330 let high = key | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
331 endif |
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
332 endfor |
21991 | 333 call add(result, [low, high]) |
334 return result | |
335 endfunc | |
336 | |
337 func Test_CharDictToPairList() | |
338 let dict = {0x1020: 1, 0x1021: 1, 0x1022: 1, | |
339 \ 0x1024: 1, | |
340 \ 0x2022: 1, | |
341 \ 0x2024: 1, 0x2025: 1} | |
342 call assert_equal([ | |
343 \ [0x1020, 0x1022], | |
344 \ [0x1024, 0x1024], | |
345 \ [0x2022, 0x2022], | |
346 \ [0x2024, 0x2025], | |
347 \ ], CharDictToPairList(dict)) | |
348 if v:errors != [] | |
349 echoerr 'CharDictToPairList' v:errors | |
350 return 1 | |
351 endif | |
352 return 0 | |
353 endfunc | |
354 | |
355 | |
356 " Build the amoji width table in a new buffer. | |
357 func BuildEmojiTable() | |
358 " First make the table for all emojis. | |
359 let pattern = '; Emoji\s\+#\s' | |
360 let lines = map(filter(filter(getline(1, '$'), 'v:val=~"^[1-9]"'), 'v:val=~pattern'), 'matchstr(v:val,"^\\S\\+")') | |
361 | |
362 " Make a dictionary with an entry for each character. | |
363 let chardict = {} | |
364 call AddLinesToCharDict(lines, chardict) | |
365 let pairlist = CharDictToPairList(chardict) | |
366 let allranges = map(pairlist, 'printf(" {0x%04x, 0x%04x},", v:val[0], v:val[1])') | |
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
367 |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
368 " New buffer to put the result in. |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
369 new |
21991 | 370 exe 'file emoji_all' |
371 call setline(1, "static struct interval emoji_all[] =") | |
372 call setline(2, "{") | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
373 call append('$', allranges) |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
374 call setline('$', getline('$')[:-2]) " remove last comma |
21991 | 375 call setline(line('$') + 1, "};") |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
376 wincmd p |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
377 |
21991 | 378 " Make the table for wide emojis. |
379 let pattern = '; Emoji_\(Presentation\|Modifier_Base\)\s\+#\s' | |
380 let lines = map(filter(filter(getline(1, '$'), 'v:val=~"^[1-9]"'), 'v:val=~pattern'), 'matchstr(v:val,"^\\S\\+")') | |
381 | |
382 " Make a dictionary with an entry for each character. | |
383 let chardict = {} | |
384 call AddLinesToCharDict(lines, chardict) | |
385 | |
386 " exclude characters that are in the "ambiguous" or "doublewidth" table | |
387 for ambi in s:ambitable | |
388 for nr in range(ambi[0], ambi[1]) | |
389 if has_key(chardict, nr) | |
390 call remove(chardict, nr) | |
391 endif | |
392 endfor | |
393 endfor | |
394 | |
395 for wide in s:doubletable | |
396 for nr in range(wide[0], wide[1]) | |
397 if has_key(chardict, nr) | |
398 call remove(chardict, nr) | |
399 endif | |
400 endfor | |
401 endfor | |
402 | |
403 let pairlist = CharDictToPairList(chardict) | |
404 let wide_ranges = map(pairlist, 'printf("\t{0x%04x, 0x%04x},", v:val[0], v:val[1])') | |
405 | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
406 " New buffer to put the result in. |
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
407 new |
21991 | 408 exe 'file emoji_wide' |
409 call setline(1, " static struct interval emoji_wide[] =") | |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
410 call setline(2, " {") |
21991 | 411 call append('$', wide_ranges) |
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
412 call setline('$', getline('$')[:-2]) " remove last comma |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
413 call setline(line('$') + 1, " };") |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
414 wincmd p |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
415 endfunc |
2041 | 416 |
21991 | 417 " First test a few things |
418 let v:errors = [] | |
419 if Test_AddLinesToCharDict() || Test_CharDictToPairList() | |
420 finish | |
421 endif | |
422 | |
423 | |
6864 | 424 " Try to avoid hitting E36 |
425 set equalalways | |
2041 | 426 |
427 " Edit the Unicode text file. Requires the netrw plugin. | |
428 edit http://unicode.org/Public/UNIDATA/UnicodeData.txt | |
429 | |
430 " Parse each line, create a list of lists. | |
431 call ParseDataToProps() | |
432 | |
433 " Build the toLower table. | |
434 call BuildCaseTable("Lower", 13) | |
435 | |
436 " Build the toUpper table. | |
437 call BuildCaseTable("Upper", 12) | |
438 | |
439 " Build the ranges of composing chars. | |
440 call BuildCombiningTable() | |
441 | |
442 " Edit the case folding text file. Requires the netrw plugin. | |
443 edit http://www.unicode.org/Public/UNIDATA/CaseFolding.txt | |
444 | |
445 " Parse each line, create a list of lists. | |
446 call ParseFoldProps() | |
447 | |
448 " Build the foldCase table. | |
449 call BuildFoldTable() | |
450 | |
451 " Edit the width text file. Requires the netrw plugin. | |
452 edit http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt | |
453 | |
454 " Parse each line, create a list of lists. | |
455 call ParseWidthProps() | |
456 | |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
457 " Build the double width table. |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
458 let s:doubletable = [] |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
459 call BuildWidthTable('[WF]', 'doublewidth') |
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
460 |
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
461 " Build the ambiguous width table. |
8680
131e651fb347
commit https://github.com/vim/vim/commit/b86f10ee10bdf932df02bdaf601dffa671518a47
Christian Brabandt <cb@256bit.org>
parents:
8629
diff
changeset
|
462 let s:ambitable = [] |
2063
1378bc45ebe5
updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents:
2041
diff
changeset
|
463 call BuildWidthTable('A', 'ambiguous') |
8629
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
464 |
54ac275e3fc4
commit https://github.com/vim/vim/commit/3848e00e0177abdb31bc600234967863ec487233
Christian Brabandt <cb@256bit.org>
parents:
6864
diff
changeset
|
465 " Edit the emoji text file. Requires the netrw plugin. |
30503
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
466 " commented out, because it drops too many characters |
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
467 "edit https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt |
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
468 " |
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
469 "" Build the emoji table. Ver. 1.0 - 6.0 |
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
470 "" Must come after the "ambiguous" and "doublewidth" tables |
3ee335235412
patch 9.0.0587: Unicode tables are outdated
Bram Moolenaar <Bram@vim.org>
parents:
25062
diff
changeset
|
471 "call BuildEmojiTable() |