2# Copyright (C) 2004 Brion Vibber <brion@pobox.com>
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
30if( php_sapi_name() !=
'cli' ) {
31 die(
"Run me from the command line please.\n" );
34require_once
'UtfNormalUtil.php';
36$in = fopen(
"DerivedNormalizationProps.txt",
"rt" );
38 print "Can't open DerivedNormalizationProps.txt for reading.\n";
39 print "If necessary, fetch this file from the internet:\n";
40 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
43print "Initializing normalization quick check tables...\n";
45while(
false !== ($line = fgets(
$in ) ) ) {
46 if( preg_match(
'/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
47 list( $junk, $first, $last, $prop, $value ) = $matches;
48 #print "$first $last $prop $value\n";
49 if( !$last ) $last = $first;
50 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
58$in = fopen(
"CompositionExclusions.txt",
"rt" );
60 print "Can't open CompositionExclusions.txt for reading.\n";
61 print "If necessary, fetch this file from the internet:\n";
62 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
66while(
false !== ($line = fgets(
$in ) ) ) {
67 if( preg_match(
'/^([0-9A-F]+)/i', $line, $matches ) ) {
68 $codepoint = $matches[1];
75$in = fopen(
"UnicodeData.txt",
"rt" );
77 print "Can't open UnicodeData.txt for reading.\n";
78 print "If necessary, fetch this file from the internet:\n";
79 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
91print "Reading character definitions...\n";
92while(
false !== ($line = fgets(
$in ) ) ) {
96 $canonicalCombiningClass =
$columns[3];
101 if( $canonicalCombiningClass != 0 ) {
105 if( $decompositionMapping ===
'' )
continue;
106 if( preg_match(
'/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
107 # Compatibility decomposition
109 $decompositionMapping = $matches[2];
125 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
129print "Recursively expanding canonical mappings...\n";
133 print "pass $pass\n";
136 $newDest = preg_replace_callback(
137 '/([\xc0-\xff][\x80-\xbf]+)/',
140 if( $newDest === $dest )
continue;
147print "Recursively expanding compatibility mappings...\n";
151 print "pass $pass\n";
154 $newDest = preg_replace_callback(
155 '/([\xc0-\xff][\x80-\xbf]+)/',
158 if( $newDest === $dest )
continue;
165print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
167$out = fopen(
"UtfNormalData.inc",
"wt");
173 $outdata =
"<" .
"?php
180global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCheckNFC;
181\$utfCombiningClass = unserialize( '$serCombining' );
182\$utfCanonicalComp = unserialize( '$serComp' );
183\$utfCanonicalDecomp = unserialize( '$serCanon' );
184\$utfCheckNFC = unserialize( '$serCheckNFC' );
186 fputs(
$out, $outdata );
188 print "Wrote out UtfNormalData.inc\n";
190 print "Can't create file UtfNormalData.inc\n";
195$out = fopen(
"UtfNormalDataK.inc",
"wt");
198 $outdata =
"<" .
"?php
205global \$utfCompatibilityDecomp;
206\$utfCompatibilityDecomp = unserialize( '$serCompat' );
208 fputs(
$out, $outdata );
210 print "Wrote out UtfNormalDataK.inc\n";
213 print "Can't create file UtfNormalDataK.inc\n";
callbackCanonical( $matches)
callbackCompat( $matches)
if(! $in) $compatibilityDecomp
escapeSingleString( $string)
Escape a string for inclusion in a PHP single-quoted string literal.
hexSequenceToUtf8( $sequence)
Take a series of space-separated hexadecimal numbers representing Unicode code points and return a UT...
codepointToUtf8( $codepoint)
Return UTF-8 sequence for a given Unicode code point.