ILIAS  release_5-1 Revision 5.0.0-5477-g43f3e3fab5f
UtfNormalGenerate.php
Go to the documentation of this file.
1<?php
2# Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3# http://www.mediawiki.org/
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18# http://www.gnu.org/copyleft/gpl.html
19
30if( php_sapi_name() != 'cli' ) {
31 die( "Run me from the command line please.\n" );
32}
33
34require_once 'UtfNormalUtil.php';
35
36$in = fopen("DerivedNormalizationProps.txt", "rt" );
37if( !$in ) {
38 print "Can't open DerivedNormalizationProps.txt for reading.\n";
39 print "If necessary, fetch this file from the internet:\n";
40 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
41 exit(-1);
42}
43print "Initializing normalization quick check tables...\n";
44$checkNFC = array();
45while( false !== ($line = fgets( $in ) ) ) {
46 if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
47 list( $junk, $first, $last, $prop, $value ) = $matches;
48 #print "$first $last $prop $value\n";
49 if( !$last ) $last = $first;
50 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
51 $char = codepointToUtf8( $i );
52 $checkNFC[$char] = $value;
53 }
54 }
55}
56fclose( $in );
57
58$in = fopen("CompositionExclusions.txt", "rt" );
59if( !$in ) {
60 print "Can't open CompositionExclusions.txt for reading.\n";
61 print "If necessary, fetch this file from the internet:\n";
62 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
63 exit(-1);
64}
65$exclude = array();
66while( false !== ($line = fgets( $in ) ) ) {
67 if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
68 $codepoint = $matches[1];
69 $source = codepointToUtf8( hexdec( $codepoint ) );
70 $exclude[$source] = true;
71 }
72}
73fclose($in);
74
75$in = fopen("UnicodeData.txt", "rt" );
76if( !$in ) {
77 print "Can't open UnicodeData.txt for reading.\n";
78 print "If necessary, fetch this file from the internet:\n";
79 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
80 exit(-1);
81}
82
90
91print "Reading character definitions...\n";
92while( false !== ($line = fgets( $in ) ) ) {
93 $columns = split(';', $line);
94 $codepoint = $columns[0];
95 $name = $columns[1];
96 $canonicalCombiningClass = $columns[3];
97 $decompositionMapping = $columns[5];
98
99 $source = codepointToUtf8( hexdec( $codepoint ) );
100
101 if( $canonicalCombiningClass != 0 ) {
102 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
103 }
104
105 if( $decompositionMapping === '' ) continue;
106 if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
107 # Compatibility decomposition
108 $canonical = false;
109 $decompositionMapping = $matches[2];
110 $compat++;
111 } else {
112 $canonical = true;
113 $canon++;
114 }
115 $total++;
116 $dest = hexSequenceToUtf8( $decompositionMapping );
117
118 $compatibilityDecomp[$source] = $dest;
119 if( $canonical ) {
120 $canonicalDecomp[$source] = $dest;
121 if( empty( $exclude[$source] ) ) {
122 $canonicalComp[$dest] = $source;
123 }
124 }
125 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
126}
127fclose( $in );
128
129print "Recursively expanding canonical mappings...\n";
132while( $changed > 0 ) {
133 print "pass $pass\n";
134 $changed = 0;
135 foreach( $canonicalDecomp as $source => $dest ) {
136 $newDest = preg_replace_callback(
137 '/([\xc0-\xff][\x80-\xbf]+)/',
138 'callbackCanonical',
139 $dest);
140 if( $newDest === $dest ) continue;
141 $changed++;
142 $canonicalDecomp[$source] = $newDest;
143 }
144 $pass++;
145}
146
147print "Recursively expanding compatibility mappings...\n";
148$changed = 42;
149$pass = 1;
150while( $changed > 0 ) {
151 print "pass $pass\n";
152 $changed = 0;
153 foreach( $compatibilityDecomp as $source => $dest ) {
154 $newDest = preg_replace_callback(
155 '/([\xc0-\xff][\x80-\xbf]+)/',
156 'callbackCompat',
157 $dest);
158 if( $newDest === $dest ) continue;
159 $changed++;
160 $compatibilityDecomp[$source] = $newDest;
161 }
162 $pass++;
163}
164
165print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
166
167$out = fopen("UtfNormalData.inc", "wt");
168if( $out ) {
169 $serCombining = escapeSingleString( serialize( $combiningClass ) );
170 $serComp = escapeSingleString( serialize( $canonicalComp ) );
171 $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
172 $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
173 $outdata = "<" . "?php
180global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCheckNFC;
181\$utfCombiningClass = unserialize( '$serCombining' );
182\$utfCanonicalComp = unserialize( '$serComp' );
183\$utfCanonicalDecomp = unserialize( '$serCanon' );
184\$utfCheckNFC = unserialize( '$serCheckNFC' );
185?" . ">\n";
186 fputs( $out, $outdata );
187 fclose( $out );
188 print "Wrote out UtfNormalData.inc\n";
189} else {
190 print "Can't create file UtfNormalData.inc\n";
191 exit(-1);
192}
193
194
195$out = fopen("UtfNormalDataK.inc", "wt");
196if( $out ) {
197 $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
198 $outdata = "<" . "?php
205global \$utfCompatibilityDecomp;
206\$utfCompatibilityDecomp = unserialize( '$serCompat' );
207?" . ">\n";
208 fputs( $out, $outdata );
209 fclose( $out );
210 print "Wrote out UtfNormalDataK.inc\n";
211 exit(0);
212} else {
213 print "Can't create file UtfNormalDataK.inc\n";
214 exit(-1);
215}
216
217# ---------------
218
219function callbackCanonical( $matches ) {
220 global $canonicalDecomp;
221 if( isset( $canonicalDecomp[$matches[1]] ) ) {
222 return $canonicalDecomp[$matches[1]];
223 }
224 return $matches[1];
225}
226
227function callbackCompat( $matches ) {
229 if( isset( $compatibilityDecomp[$matches[1]] ) ) {
230 return $compatibilityDecomp[$matches[1]];
231 }
232 return $matches[1];
233}
234
235?>
if(! $in) $columns
Definition: Utf8Test.php:46
$canonicalDecomp
if(! $in) print
callbackCanonical( $matches)
$canonicalComp
if(! $in) $exclude
callbackCompat( $matches)
if(! $in) $compatibilityDecomp
$combiningClass
escapeSingleString( $string)
Escape a string for inclusion in a PHP single-quoted string literal.
hexSequenceToUtf8( $sequence)
Take a series of space-separated hexadecimal numbers representing Unicode code points and return a UT...
codepointToUtf8( $codepoint)
Return UTF-8 sequence for a given Unicode code point.
exit
Definition: login.php:54