annotate runtime/tools/unicode.vim @ 7539:f5d3b2c6f971

Added tag v7.4.1070 for changeset c9fc24b7629385e94b7479146e610fc7c28d07a3
author Christian Brabandt <cb@256bit.org>
date Sat, 09 Jan 2016 19:45:05 +0100
parents c9a5d51c9161
children 54ac275e3fc4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
1 " Script to extract tables from Unicode .txt files, to be used in src/mbyte.c.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
2 " The format of the UnicodeData.txt file is explained here:
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
3 " http://www.unicode.org/Public/5.1.0/ucd/UCD.html
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
4 " For the other files see the header.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
5 "
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
6 " Usage: Vim -S <this-file>
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
7 "
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
8 " Author: Bram Moolenaar
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
9 " Last Update: 2010 Jan 12
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
10
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
11 " Parse lines of UnicodeData.txt. Creates a list of lists in s:dataprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
12 func! ParseDataToProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
13 let s:dataprops = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
14 let lnum = 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
15 while lnum <= line('$')
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
16 let l = split(getline(lnum), '\s*;\s*', 1)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
17 if len(l) != 15
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
18 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 15'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
19 return
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
20 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
21 call add(s:dataprops, l)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
22 let lnum += 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
23 endwhile
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
24 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
25
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
26 " Parse lines of CaseFolding.txt. Creates a list of lists in s:foldprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
27 func! ParseFoldProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
28 let s:foldprops = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
29 let lnum = 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
30 while lnum <= line('$')
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
31 let line = getline(lnum)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
32 if line !~ '^#' && line !~ '^\s*$'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
33 let l = split(line, '\s*;\s*', 1)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
34 if len(l) != 4
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
35 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 4'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
36 return
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
37 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
38 call add(s:foldprops, l)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
39 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
40 let lnum += 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
41 endwhile
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
42 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
43
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
44 " Parse lines of EastAsianWidth.txt. Creates a list of lists in s:widthprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
45 func! ParseWidthProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
46 let s:widthprops = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
47 let lnum = 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
48 while lnum <= line('$')
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
49 let line = getline(lnum)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
50 if line !~ '^#' && line !~ '^\s*$'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
51 let l = split(line, '\s*;\s*', 1)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
52 if len(l) != 2
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
53 echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 2'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
54 return
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
55 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
56 call add(s:widthprops, l)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
57 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
58 let lnum += 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
59 endwhile
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
60 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
61
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
62 " Build the toLower or toUpper table in a new buffer.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
63 " Uses s:dataprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
64 func! BuildCaseTable(name, index)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
65 let start = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
66 let end = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
67 let step = 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
68 let add = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
69 let ranges = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
70 for p in s:dataprops
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
71 if p[a:index] != ''
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
72 let n = ('0x' . p[0]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
73 let nl = ('0x' . p[a:index]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
74 if start >= 0 && add == nl - n && (step == 0 || n - end == step)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
75 " continue with same range.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
76 let step = n - end
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
77 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
78 else
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
79 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
80 " produce previous range
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
81 call Range(ranges, start, end, step, add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
82 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
83 let start = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
84 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
85 let step = 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
86 let add = nl - n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
87 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
88 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
89 endfor
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
90 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
91 call Range(ranges, start, end, step, add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
92 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
93
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
94 " New buffer to put the result in.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
95 new
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
96 exe "file to" . a:name
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
97 call setline(1, "static convertStruct to" . a:name . "[] =")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
98 call setline(2, "{")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
99 call append('$', ranges)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
100 call setline('$', getline('$')[:-2]) " remove last comma
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
101 call setline(line('$') + 1, "};")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
102 wincmd p
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
103 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
104
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
105 " Build the foldCase table in a new buffer.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
106 " Uses s:foldprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
107 func! BuildFoldTable()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
108 let start = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
109 let end = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
110 let step = 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
111 let add = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
112 let ranges = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
113 for p in s:foldprops
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
114 if p[1] == 'C' || p[1] == 'S'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
115 let n = ('0x' . p[0]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
116 let nl = ('0x' . p[2]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
117 if start >= 0 && add == nl - n && (step == 0 || n - end == step)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
118 " continue with same range.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
119 let step = n - end
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
120 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
121 else
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
122 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
123 " produce previous range
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
124 call Range(ranges, start, end, step, add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
125 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
126 let start = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
127 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
128 let step = 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
129 let add = nl - n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
130 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
131 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
132 endfor
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
133 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
134 call Range(ranges, start, end, step, add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
135 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
136
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
137 " New buffer to put the result in.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
138 new
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
139 file foldCase
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
140 call setline(1, "static convertStruct foldCase[] =")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
141 call setline(2, "{")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
142 call append('$', ranges)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
143 call setline('$', getline('$')[:-2]) " remove last comma
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
144 call setline(line('$') + 1, "};")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
145 wincmd p
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
146 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
147
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
148 func! Range(ranges, start, end, step, add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
149 let s = printf("\t{0x%x,0x%x,%d,%d},", a:start, a:end, a:step == 0 ? -1 : a:step, a:add)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
150 call add(a:ranges, s)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
151 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
152
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
153 " Build the combining table.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
154 " Uses s:dataprops.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
155 func! BuildCombiningTable()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
156 let start = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
157 let end = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
158 let ranges = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
159 for p in s:dataprops
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
160 if p[2] == 'Mn' || p[2] == 'Mc' || p[2] == 'Me'
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
161 let n = ('0x' . p[0]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
162 if start >= 0 && end + 1 == n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
163 " continue with same range.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
164 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
165 else
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
166 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
167 " produce previous range
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
168 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
169 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
170 let start = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
171 let end = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
172 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
173 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
174 endfor
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
175 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
176 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
177 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
178
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
179 " New buffer to put the result in.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
180 new
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
181 file combining
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
182 call setline(1, " static struct interval combining[] =")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
183 call setline(2, " {")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
184 call append('$', ranges)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
185 call setline('$', getline('$')[:-2]) " remove last comma
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
186 call setline(line('$') + 1, " };")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
187 wincmd p
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
188 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
189
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
190 " Build the double width or ambiguous width table in a new buffer.
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
191 " Uses s:widthprops and s:dataprops.
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
192 func! BuildWidthTable(pattern, tableName)
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
193 let start = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
194 let end = -1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
195 let ranges = []
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
196 let dataidx = 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
197 for p in s:widthprops
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
198 if p[1][0] =~ a:pattern
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
199 if p[0] =~ '\.\.'
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
200 " It is a range. we don't check for composing char then.
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
201 let rng = split(p[0], '\.\.')
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
202 if len(rng) != 2
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
203 echoerr "Cannot parse range: '" . p[0] . "' in width table"
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
204 endif
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
205 let n = ('0x' . rng[0]) + 0
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
206 let n_last = ('0x' . rng[1]) + 0
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
207 else
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
208 let n = ('0x' . p[0]) + 0
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
209 let n_last = n
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
210 endif
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
211 " Find this char in the data table.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
212 while 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
213 let dn = ('0x' . s:dataprops[dataidx][0]) + 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
214 if dn >= n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
215 break
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
216 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
217 let dataidx += 1
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
218 endwhile
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
219 if dn != n && n_last == n
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
220 echoerr "Cannot find character " . n . " in data table"
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
221 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
222 " Only use the char when it's not a composing char.
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
223 " But use all chars from a range.
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
224 let dp = s:dataprops[dataidx]
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
225 if n_last > n || (dp[2] != 'Mn' && dp[2] != 'Mc' && dp[2] != 'Me')
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
226 if start >= 0 && end + 1 == n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
227 " continue with same range.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
228 else
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
229 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
230 " produce previous range
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
231 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
232 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
233 let start = n
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
234 endif
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
235 let end = n_last
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
236 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
237 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
238 endfor
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
239 if start >= 0
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
240 call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
241 endif
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
242
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
243 " New buffer to put the result in.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
244 new
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
245 exe "file " . a:tableName
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
246 call setline(1, " static struct interval " . a:tableName . "[] =")
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
247 call setline(2, " {")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
248 call append('$', ranges)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
249 call setline('$', getline('$')[:-2]) " remove last comma
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
250 call setline(line('$') + 1, " };")
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
251 wincmd p
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
252 endfunc
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
253
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
254
6864
c9a5d51c9161 patch 7.4.752
Bram Moolenaar <bram@vim.org>
parents: 2063
diff changeset
255 " Try to avoid hitting E36
c9a5d51c9161 patch 7.4.752
Bram Moolenaar <bram@vim.org>
parents: 2063
diff changeset
256 set equalalways
2041
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
257
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
258 " Edit the Unicode text file. Requires the netrw plugin.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
259 edit http://unicode.org/Public/UNIDATA/UnicodeData.txt
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
260
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
261 " Parse each line, create a list of lists.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
262 call ParseDataToProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
263
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
264 " Build the toLower table.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
265 call BuildCaseTable("Lower", 13)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
266
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
267 " Build the toUpper table.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
268 call BuildCaseTable("Upper", 12)
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
269
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
270 " Build the ranges of composing chars.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
271 call BuildCombiningTable()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
272
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
273 " Edit the case folding text file. Requires the netrw plugin.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
274 edit http://www.unicode.org/Public/UNIDATA/CaseFolding.txt
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
275
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
276 " Parse each line, create a list of lists.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
277 call ParseFoldProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
278
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
279 " Build the foldCase table.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
280 call BuildFoldTable()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
281
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
282 " Edit the width text file. Requires the netrw plugin.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
283 edit http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
284
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
285 " Parse each line, create a list of lists.
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
286 call ParseWidthProps()
d5867fd6b2b7 updated for version 7.2.330
Bram Moolenaar <bram@zimbu.org>
parents:
diff changeset
287
2063
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
288 " Build the double width table.
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
289 call BuildWidthTable('[WF]', 'doublewidth')
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
290
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
291 " Build the ambiguous width table.
1378bc45ebe5 updated for version 7.2.348
Bram Moolenaar <bram@zimbu.org>
parents: 2041
diff changeset
292 call BuildWidthTable('A', 'ambiguous')