<?php
/* Generate ECI multibyte tables from unicode.org mapping files */
/*
    libzueci - an open source UTF-8 ECI library adapted from libzint
    Copyright (C) 2022 gitlost
*/
/* SPDX-License-Identifier: BSD-3-Clause */

/*
 * To create "zueci_big5/gb18030/gb2312/gbk/ksx1001/sjis.h" (from project root directory):
 *
 *   php tools/gen_zueci_mb_h.php
 *
 * NOTE: tools/data/GB18030.TXT will have to be downloaded first from the tarball
 *       https://haible.de/bruno/charsets/conversion-tables/GB18030.tar.bz2
 *       using the version jdk-1.4.2/GB18030.TXT
 *
 * NOTE: tools/data/GB2312.TXT will have to be downloaded first from the tarball
 *       https://haible.de/bruno/charsets/conversion-tables/GB2312.tar.bz2
 *       using the version unicode.org-mappings/EASTASIA/GB/GB2312.TXT
 */
// 'zend.assertions' should set to 1 in php.ini

$copyright_text = <<<'EOD'
/* SPDX-License-Identifier: BSD-3-Clause */

EOD;

$basename = basename(__FILE__);
$dirname = dirname(__FILE__);

$opts = getopt('d:o:');
$data_dirname = isset($opts['d']) ? $opts['d'] : ($dirname . '/data'); // Where to load file from.
$out_dirname = isset($opts['o']) ? $opts['o'] : ($dirname . '/..'); // Where to put output.

$year = 2022;

function out_header(&$out, $name, $descr, $file, $start_year = 0, $extra_comment = '') {
    global $copyright_text, $basename, $year;
    $caps = strtoupper($name);
    $out[] = '/*  ' . $name . '.h - tables for Unicode to/from ' . $descr . ', generated by "tools/' . $basename . '" from';
    if ($extra_comment !== '') {
        $out[] = '    "' . $file . '"';
        $out[] = '    ' . $extra_comment . ' */';
    } else {
        $out[] = '    "' . $file . '" */';
    }
    $out[] = '/*';
    $out[] = '    libzueci - an open source UTF-8 ECI library adapted from libzint';
    if ($start_year && $start_year != $year) {
        $out[] = '    Copyright (C) ' . $start_year . '-' . $year . ' gitlost';
    } else {
        $out[] = '    Copyright (C) ' . $year . ' gitlost';
    }
    $out[] = ' */';
    $out = array_merge($out, explode("\n", $copyright_text));
    $out[] = '#ifndef ZUECI_' . $caps . '_H';
    $out[] = '#define ZUECI_' . $caps . '_H';
}

/* Output a block of table entries to `$out` array */
function out_tab_entries(&$out, $arr, $cnt, $not_hex = false) {
    $line = '   ';
    for ($i = 0; $i < $cnt; $i++) {
        if ($i && $i % 8 === 0) {
            $out[] = $line;
            $line = '   ';
        }
        if ($not_hex) {
            $line .= sprintf(' %5d,', $arr[$i]);
        } else {
            $line .= sprintf(' 0x%04X,', $arr[$i]);
        }
    }
    if ($line !== '   ') {
        $out[] = $line;
    }
}

/* Output tables to `$out` array */
function out_tabs(&$out, $name, $u_u, $u_mb, $mb_u, $no_u_ind = false, $u_u_comment = '', $u_mb_comment = '', $mb_u_comment = '') {
    if ($u_u_comment == '') $u_u_comment = 'Unicode codepoints sorted';
    $cnt_u_u = count($u_u);
    $out[] = '';
    $out[] = '#ifndef ZUECI_EMBED_NO_TO_ECI';
    $out[] = '/* ' . $u_u_comment . ' */';
    $out[] = 'static const zueci_u16 zueci_' . $name . '_u_u[' . $cnt_u_u . '] = {';
    out_tab_entries($out, $u_u, $cnt_u_u);
    $out[] = '};';
    $out[] = '#endif /* ZUECI_EMBED_NO_TO_ECI */';

    if (!empty($u_mb)) {
        if ($u_mb_comment == '') $u_mb_comment = 'Multibyte values sorted in Unicode order';
        $cnt = count($u_mb);
        $out[] = '';
        $out[] = '#ifndef ZUECI_EMBED_NO_TO_ECI';
        $out[] = '/* ' . $u_mb_comment . ' */';
        $out[] = 'static const zueci_u16 zueci_' . $name . '_u_mb[' . $cnt . '] = {';
        $line = '   ';
        out_tab_entries($out, $u_mb, $cnt);
        $out[] = '};';
        $out[] = '#endif /* ZUECI_EMBED_NO_TO_ECI */';
    }
    if (!$no_u_ind) {
        $ind_cnt = ($u_u[$cnt_u_u - 1] >> 8) + 1;
        $out[] = '';
        $out[] = '#ifndef ZUECI_EMBED_NO_TO_ECI';
        $out[] = '/* Indexes into Unicode `zueci_' . $name . '_u_u[]` array in blocks of 0x100 */';
        $ind_idx = count($out);
        $out[] = 'static const zueci_u16 zueci_' . $name . '_u_ind[] = {';
        $line = '   ';
        $i = 0;
        foreach ($u_u as $ind => $u) {
            $div = ($u - $u_u[0]) >> 8;
            while ($div >= $i) {
                if ($i && $i % 8 === 0) {
                    $out[] = $line;
                    $line = '   ';
                }
                $line .= sprintf(' %5d,', $ind);
                $i++;
            }
        }
        if ($line !== '   ') {
            $out[] = $line;
            $line = '   ';
        }
        $out[] = '};';
        $out[] = '#endif /* ZUECI_EMBED_NO_TO_ECI */';
        $out[$ind_idx] = 'static const zueci_u16 zueci_' . $name . '_u_ind[' . $i . '] = {';
    }
    if (!empty($mb_u)) {
        if ($mb_u_comment == '') $mb_u_comment = 'Unicode values in multibyte order';
        $cnt = count($mb_u);
        $out[] = '';
        $out[] = '#ifndef ZUECI_EMBED_NO_TO_UTF8';
        $out[] = '/* ' . $mb_u_comment . ' */';
        $out[] = 'static const zueci_u16 zueci_' . $name . '_mb_u[' . $cnt . '] = {';
        $line = '   ';
        out_tab_entries($out, $mb_u, $cnt);
        $out[] = '};';
        $out[] = '#endif /* ZUECI_EMBED_NO_TO_UTF8 */';
    }
}

/* Helper to output special-case URO (Unified Repertoire and Ordering) block (U+4E00-U+9FFF) tables */
function out_uro_tabs(&$out, $name, $tab_uro_u, $tab_uro_mb_ind) {
    $cnt = count($tab_uro_u);
    $out[] = '';
    $out[] = '#ifndef ZUECI_EMBED_NO_TO_ECI';
    $out[] = '/* Unicode usage bit-flags for URO (U+4E00-U+9FFF) block */';
    $out[] = 'static const zueci_u16 zueci_' . $name . '_uro_u[' . $cnt . '] = {';
    out_tab_entries($out, $tab_uro_u, $cnt);
    $cnt = count($tab_uro_mb_ind);
    $out[] = '};';
    $out[] = '';
    $out[] = '/* Multibyte indexes for URO (U+4E00-U+9FFF) block */';
    $out[] = 'static const zueci_u16 zueci_' . $name . '_uro_mb_ind[' . $cnt . '] = {';
    out_tab_entries($out, $tab_uro_mb_ind, $cnt, true /*not_hex*/);
    $out[] = '};';
    $out[] = '#endif /* ZUECI_EMBED_NO_TO_ECI */';
}

// BIG5

$out = array();

out_header($out, 'big5', 'ECI 28 Big5', 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT');

//$file = $data_dirname . '/' . 'BIG5.TXT';
$file = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us = array();
$mbs = array();
foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{4})[ \t]+0x([0-9A-F]{4})[ \t].*$/', $line, $matches)) {
        $d = hexdec($matches[1]);
        $u = hexdec($matches[2]);
        $us[] = $u;
        $mbs[] = $d;
    }
}

$u_mb = $mbs;
array_multisort($us, $u_mb);

// Calculate URO (U+4E00-U+9FFF) table
for ($u_i = 0, $cnt = count($us); $u_i < $cnt && $us[$u_i] < 0x4E00; $u_i++);

$start_u_i = $u_i;
$big5_uro_u = $big5_uro_mb_ind = array();
$sort_search = array_flip($us);
for ($u = 0x4E00; $u <= 0x9FAF; $u += 16) {
    $used = 0;
    $next_u_i = $u_i;
    for ($j = 0; $j < 16; $j++) {
        if (isset($sort_search[$u + $j])) {
            $i = $sort_search[$u + $j];
            $used |= 1 << $j;
            $next_u_i = $i + 1;
            $end_u_i = $i;
        }
    }
    $big5_uro_u[] = $used;
    $big5_uro_mb_ind[] = $u_i;
    $u_i = $next_u_i;
}

// Output URO tables
out_uro_tabs($out, 'big5', $big5_uro_u, $big5_uro_mb_ind);

// Remove URO block from Unicode table
$u_u = $us;
array_splice($u_u, $start_u_i, $end_u_i - $start_u_i + 1);

// Multibyte ordered array
$mb_u = array();
$u_mb_search = array_flip($u_mb);

for ($i = 0xA1; $i < 0xFA; $i++) {
	if ($i == 0xC8) $i = 0xC9;

	for ($j = 0x40; $j < 0xFF; $j++) {
		if ($j == 0x7F) $j = 0xA1;

		$k = ($i << 8) | $j;
		if (isset($u_mb_search[$k])) {
			$mb_u[] = $us[$u_mb_search[$k]];
		} else {
			$mb_u[] = 0;
		}
		if ($i == 0xF9 && $j == 0xD5) break;
	}
}

// Output Big5 tables

out_tabs($out, 'big5', $u_u, $u_mb, $mb_u, true /*no_ind*/);

$out[] = '';
$out[] = '#endif /* ZUECI_BIG5_H */';

file_put_contents($out_dirname . '/zueci_big5.h', implode("\n", $out) . "\n");

// EUC-KR (KS X 1001)

$out = array();

out_header($out, 'ksx1001', 'ECI 30 EUC-KR (KS X 1001)',
            'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/KSX1001.TXT');

//$file =  $data_dirname . '/' . 'KSX1001.TXT';
$file = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/KSX1001.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us = array();
$mbs = array();
foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{4})[ \t]+0x([0-9A-F]{4})[ \t].*$/', $line, $matches)) {
        $d = hexdec($matches[1]) + 0x8080; // Convert to EUC-KR
        $u = hexdec($matches[2]);
        $us[] = $u;
        $mbs[] = $d;
    }
}

// Add some characters defined later than in KSX1001.TXT

$us[] = 0x20AC; // Euro sign added KS X 1001:1998
$mbs[] = 0x2266 + 0x8080;

$us[] = 0xAE; // Registered trademark added KS X 1001:1998
$mbs[] = 0x2267 + 0x8080;

$us[] = 0x327E; // Korean postal code symbol added KS X 1001:2002
$mbs[]= 0x2268 + 0x8080;

$u_mb = $mbs;
array_multisort($us, $u_mb);

// Calculate URO (U+4E00-U+9FFF) table
for ($u_i = 0, $cnt = count($us); $u_i < $cnt && $us[$u_i] < 0x4E00; $u_i++);

$start_u_i = $u_i;
$ksx1001_uro_u = $ksx1001_uro_mb_ind = array();
$sort_search = array_flip($us);
for ($u = 0x4E00; $u <= 0x9F9F; $u += 16) {
    $used = 0;
    $next_u_i = $u_i;
    for ($j = 0; $j < 16; $j++) {
        if (isset($sort_search[$u + $j])) {
            $i = $sort_search[$u + $j];
            $used |= 1 << $j;
            $next_u_i = $i + 1;
            $end_u_i = $i;
        }
    }
    $ksx1001_uro_u[] = $used;
    $ksx1001_uro_mb_ind[] = $u_i;
    $u_i = $next_u_i;
}

// Output URO tables
out_uro_tabs($out, 'ksx1001', $ksx1001_uro_u, $ksx1001_uro_mb_ind);

// Remove URO block from Unicode table
$u_u = $us;
array_splice($u_u, $start_u_i, $end_u_i - $start_u_i + 1);

// Multibyte ordered array
$mb_u = array();
$u_mb_search = array_flip($u_mb);

for ($i = 0xA1; $i < 0xFE; $i++) {
	if ($i == 0xAD) {
		$i = 0xB0;
	} else if ($i == 0xC9) {
		$i = 0xCA;
	}

	for ($j = 0xA1; $j < 0xFF; $j++) {

		$k = ($i << 8) | $j;
		if (isset($u_mb_search[$k])) {
			$mb_u[] = $us[$u_mb_search[$k]];
		} else {
			$mb_u[] = 0;
		}
	}
}

// Output KS X 1001 tables
out_tabs($out, 'ksx1001', $u_u, $u_mb, $mb_u);

$out[] = '';
$out[] = '#endif /* ZUECI_KSX1001_H */';

file_put_contents($out_dirname . '/zueci_ksx1001.h', implode("\n", $out) . "\n");

// Shift JIS

$out = array();

out_header($out, 'sjis', 'ECI 20 Shift JIS', 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/SHIFTJIS.TXT');

//$file = $data_dirname . '/' . 'SHIFTJIS.TXT';
$file = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/SHIFTJIS.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us = array();
$mbs = array();
foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{2,4})[ \t]+0x([0-9A-F]{4})[ \t].*$/', $line, $matches)) {
        $d = hexdec($matches[1]);
        if ($d < 0x80 && $d != 0x5C && $d != 0x7E) {
            continue;
        }
        $u = hexdec($matches[2]);
        // PUA characters (user-defined range), dealt with programatically by `zueci_u_sjis()`
        // See CJKV Information Processing by Ken Lunde, 2nd ed., Table 4-86, p.286
        // https://file.allitebooks.com/20160708/CJKV%20Information%20Processing.pdf
        if ($u >= 0xE000 && $u <= 0xE757) {
            continue;
        }
        $us[] = $u;
        $mbs[] = $d;
    }
}

$u_mb = $mbs;
array_multisort($us, $u_mb);

// Calculate URO (U+4E00-U+9FFF) table
for ($u_i = 0, $cnt = count($us); $u_i < $cnt && $us[$u_i] < 0x4E00; $u_i++);

$start_u_i = $u_i;
$sjis_uro_u = $sjis_uro_mb_ind = array();
$sort_search = array_flip($us);
for ($u = 0x4E00; $u <= 0x9FAF; $u += 16) {
    $used = 0;
    $next_u_i = $u_i;
    for ($j = 0; $j < 16; $j++) {
        if (isset($sort_search[$u + $j])) {
            $i = $sort_search[$u + $j];
            $used |= 1 << $j;
            $next_u_i = $i + 1;
            $end_u_i = $i;
        }
    }
    $sjis_uro_u[] = $used;
    $sjis_uro_mb_ind[] = $u_i;
    $u_i = $next_u_i;
}

// Output URO tables
out_uro_tabs($out, 'sjis', $sjis_uro_u, $sjis_uro_mb_ind);

// Remove URO block from Unicode table
$u_u = $us;
array_splice($u_u, $start_u_i, $end_u_i - $start_u_i + 1);

// Multibyte ordered array
$mb_u = array();
$u_mb_search = array_flip($u_mb);

for ($i = 0x81; $i < 0xEB; $i++) {
	if ($i == 0xA0) {
		$i = 0xE0;
	}

	for ($j = 0x40; $j < 0xFD; $j++) {
		if ($j == 0x7F) $j = 0x80;

		$k = ($i << 8) | $j;
		if (isset($u_mb_search[$k])) {
			$mb_u[] = $us[$u_mb_search[$k]];
		} else {
			$mb_u[] = 0;
		}

		if ($i == 0xEA && $j == 0xA4) break;
	}
}

// Output Shift JIS tables
out_tabs($out, 'sjis', $u_u, $u_mb, $mb_u, true /*no_ind*/);

$out[] = '';
$out[] = '#endif /* ZUECI_SJIS_H */';

file_put_contents($out_dirname . '/zueci_sjis.h', implode("\n", $out) . "\n");

// GB 2312

$out = array();

out_header($out, 'gb2312', 'ECI 29 GB 2312-1980 (EUC-CN)',
            'unicode.org-mappings/EASTASIA/GB/GB2312.TXT', 0,
            '(see https://haible.de/bruno/charsets/conversion-tables/GB2312.tar.bz2)');

$file = $data_dirname . '/' . 'GB2312.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us = array();
$mbs = array();
$in_gb2312 = array();
foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{2,4})[ \t]+0x([0-9A-F]{4})[ \t].*$/', $line, $matches)) {
        $d = hexdec($matches[1]);
        if ($d < 0x80) {
            continue;
        }
        $u = hexdec($matches[2]);
        $us[] = $u;
        $mbs[] = $d + 0x8080; // Convert to EUC-CN
        $in_gb2312[$u] = true;
    }
}

$u_mb = $mbs;
array_multisort($us, $u_mb);

// Calculate URO (U+4E00-U+9FFF) table
for ($u_i = 0, $cnt = count($us); $u_i < $cnt && $us[$u_i] < 0x4E00; $u_i++);

$start_u_i = $u_i;
$gb2312_uro_u = $gb2312_uro_mb_ind = array();
$sort_search = array_flip($us);
for ($u = 0x4E00; $u <= 0x9CEF; $u += 16) {
    $used = 0;
    $next_u_i = $u_i;
    for ($j = 0; $j < 16; $j++) {
        if (isset($sort_search[$u + $j])) {
            $i = $sort_search[$u + $j];
            $used |= 1 << $j;
            $next_u_i = $i + 1;
            $end_u_i = $i;
        }
    }
    $gb2312_uro_u[] = $used;
    $gb2312_uro_mb_ind[] = $u_i;
    $u_i = $next_u_i;
}

// Output URO tables
out_uro_tabs($out, 'gb2312', $gb2312_uro_u, $gb2312_uro_mb_ind);

// Remove URO block from Unicode table
$u_u = $us;
array_splice($u_u, $start_u_i, $end_u_i - $start_u_i + 1);

// Multibyte ordered array
$mb_u = array();
$u_mb_search = array_flip($u_mb);

for ($i = 0xA1; $i < 0xF8; $i++) {
	if ($i == 0xAA) {
		$i = 0xB0;
	}

	for ($j = 0xA1; $j < 0xFF; $j++) {

		$k = ($i << 8) | $j;
		if (isset($u_mb_search[$k])) {
			$mb_u[] = $us[$u_mb_search[$k]];
		} else {
			$mb_u[] = 0;
		}
	}
}

// Output GB 2312 tables
out_tabs($out, 'gb2312', $u_u, $u_mb, $mb_u);

$out[] = '';
$out[] = '#endif /* ZUECI_GB2312_H */';

file_put_contents($out_dirname . '/zueci_gb2312.h', implode("\n", $out) . "\n");

// GBK

$out = array();

out_header($out, 'gbk', 'ECI 31 GBK (excl. GB 2312)',
            'https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP936.TXT');

//$file =  $data_dirname . '/' . 'CP936.TXT';
$file = 'https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP936.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us = array();
$mbs = array();
$in_gbk = array();
foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{2,4})[ \t]+0x([0-9A-F]{4})[ \t].*$/', $line, $matches)) {
        $d = hexdec($matches[1]);
        if ($d <= 0x80) { // Ignore weird 0x80 mapping to U+20AC (EURO SIGN) if any (present in Unicode Public mapping file)
            continue;
        }
        $u = hexdec($matches[2]);
        $in_gbk[$u] = true;
        if ($u != 0x2015 && isset($in_gb2312[$u])) { // U+2015 mapped differently by GBK
            continue;
        }
        $us[] = $u;
        $mbs[] = $d;
    }
}

$u_mb = $mbs;
array_multisort($us, $u_mb);

// Calculate URO (U+4E00-U+9FFF) table
for ($u_i = 0, $cnt = count($us); $u_i < $cnt && $us[$u_i] < 0x4E00; $u_i++);

$start_u_i = $u_i;
$gbk_uro_u = $gbk_uro_mb_ind = array();
$sort_search = array_flip($us);
for ($u = 0x4E00; $u <= 0x9FAF; $u += 16) {
    $used = 0;
    $next_u_i = $u_i;
    for ($j = 0; $j < 16; $j++) {
        if (isset($sort_search[$u + $j])) {
            $i = $sort_search[$u + $j];
            $used |= 1 << $j;
            $next_u_i = $i + 1;
            $end_u_i = $i;
        }
    }
    $gbk_uro_u[] = $used;
    $gbk_uro_mb_ind[] = $u_i;
    $u_i = $next_u_i;
}

// Output URO tables
out_uro_tabs($out, 'gbk', $gbk_uro_u, $gbk_uro_mb_ind);

// Remove URO block from Unicode table
$u_u = $us;
array_splice($u_u, $start_u_i, $end_u_i - $start_u_i + 1);

// Multibyte ordered array
$mb_u = array();
$u_mb_search = array_flip($u_mb);

$max_j = 0xFF;
for ($i = 0x81; $i < 0xFF; $i++) {
	if ($i == 0xA1) {
		$i = 0xA8;
		$max_j = 0xA1;
	}

	for ($j = 0x40; $j < $max_j; $j++) {
		if ($j == 0x7F) $j = 0x80;

		$k = ($i << 8) | $j;
		if (isset($u_mb_search[$k])) {
			$mb_u[] = $us[$u_mb_search[$k]];
		} else {
			$mb_u[] = 0;
		}

		if ($i == 0xFE && $j == 0x4F) break;
	}
}

// Output GBK tables
out_tabs($out, 'gbk', $u_u, $u_mb, $mb_u, true /*no_ind*/);

// Remove URO block from multibyte table
array_splice($u_mb, $start_u_i, $end_u_i - $start_u_i + 1);
// Sort into multibyte order
array_multisort($u_mb, $u_u);

$cnt = count($u_mb);
$out[] = '';
$out[] = '#ifndef ZUECI_EMBED_NO_TO_UTF8';
$out[] = '/* Non-URO multibyte values in multibyte order */';
$out[] = 'static const zueci_u16 zueci_gbk_nonuro_mb[' . $cnt . '] = {';
out_tab_entries($out, $u_mb, $cnt);
$out[] = '};';
$out[] = '';
$out[] = '/* Non-URO Unicode values in multibyte order */';
$out[] = 'static const zueci_u16 zueci_gbk_nonuro_u[' . $cnt . '] = {';
out_tab_entries($out, $u_u, $cnt);
$out[] = '};';
$out[] = '#endif /* ZUECI_EMBED_NO_TO_UTF8 */';

$out[] = '';
$out[] = '#endif /* ZUECI_GBK_H */';

file_put_contents($out_dirname . '/zueci_gbk.h', implode("\n", $out) . "\n");

// GB 18030

$out = array();

out_header($out, 'gb18030', 'ECI 32 GB 18030-2005', 'jdk-1.4.2/GB18030.TXT', 0,
            '(see https://haible.de/bruno/charsets/conversion-tables/GB18030.tar.bz2)');

$file = $data_dirname . '/' . 'GB18030.TXT';

// Read the file.

if (($get = file_get_contents($file)) === false) {
    error_log($error = "$basename: ERROR: Could not read mapping file \"$file\"");
    exit($error . PHP_EOL);
}

$lines = explode("\n", $get);

// Parse the file.

$us2 = array();
$mbs2 = array();
$us4 = array();
$mbs4 = array();

foreach ($lines as $line) {
    $line = trim($line);
    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
        continue;
    }
    if (preg_match('/^0x([0-9A-F]{2,8})[ \t]+0x([0-9A-F]{5})/', $line)) { // Exclude U+10000..10FFFF to save space
        continue;
    }
    $matches = array();
    if (preg_match('/^0x([0-9A-F]{2,8})[ \t]+0x([0-9A-F]{4}).*$/', $line, $matches)) {
        $d = hexdec($matches[1]);
        if ($d < 0x80) {
            continue;
        }
        $u = hexdec($matches[2]);
        // 2-byte extension GB 18030-2005 change, were PUA, see Table 3-37, p.109, Lunde 2nd ed.
        if (($u >= 0x9FB4 && $u <= 0x9FBB) || ($u >= 0xFE10 && $u <= 0xFE19)) {
            //continue;
        }
        // 4-byte extension change, PUA
        if ($u == 0xE7C7) {
            continue;
        }
        if ($d < 0x10000) {
            if (isset($in_gbk[$u])) {
                continue;
            }
            // User-defined, dealt with programatically by `zueci_u_gb18030()`
            if ($u >= 0xE000 && $u <= 0xE765) {
                continue;
            }
            $us2[] = $u;
            $mbs2[] = $d;
        } else if ($u < 0x10000) {
            $us4[] = $u;
            $mbs4[] = $d;
        }
    }
}

/* 2-byte extension GB 18030-2005 change, was PUA U+E7C7 below, see Table 3-39, p.111, Lunde 2nd ed. */
$us2[] = 0x1E3F; $mbs2[] = 0xA8BC;

/* 2-byte extension GB 18030-2005 change, were PUA, see Table 3-37, p.109, Lunde 2nd ed. */
$pua_change_2005_mb = array(
    0xFE59, 0xFE61, 0xFE66, 0xFE67, 0xFE6D, 0xFE7E, 0xFE90, 0xFEA0,
    0xA6D9, 0xA6DB, 0xA6DA, 0xA6DC, 0xA6DD, 0xA6DE, 0xA6DF, 0xA6EC, 0xA6ED, 0xA6F3,
);
$pua_change_2005_u = array(
    0x9FB4, 0x9FB5, 0x9FB6, 0x9FB7, 0x9FB8, 0x9FB9, 0x9FBA, 0x9FBB,
    0xFE10, 0xFE11, 0xFE12, 0xFE13, 0xFE14, 0xFE15, 0xFE16, 0xFE17, 0xFE18, 0xFE19,
);
for ($i = 0, $cnt = count($pua_change_2005_mb); $i < $cnt; $i++) {
    $mbs2[] = $pua_change_2005_mb[$i];
    $us2[] = $pua_change_2005_u[$i];
}

/* 4-byte extension PUA */
// Dealt with by `zueci_u_gb18030()`
//$us4[] = 0xE7C7;
//$mbs4[] = 0x8135F437;

// Calculate Unicode start/end codepoints mapping to consecutive 4-byte blocks

$u_mb4 = $mbs4;
array_multisort($us4, $u_mb4);

$gb18030_4_u_b = array();
$gb18030_4_u_e = array();
$gb18030_4_mb_e = array();
$gb18030_4_u_mb_o = array();

// Start/end points
$prev_u = $begin_u = $us4[0];
$prev_mb = $u_mb4[0];
for ($i = 1, $cnt = count($us4); $i < $cnt; $i++) {
    $u = $us4[$i];
    if ($u === $prev_u + 1) {
        $prev_u++;
        $prev_mb = $u_mb4[$i];
        continue;
    }
    $gb18030_4_u_b[] = $begin_u;
    $gb18030_4_u_e[] = $prev_u;
    $gb18030_4_mb_e[] = $prev_mb;
    $begin_u = $prev_u = $u;
    $prev_mb = $u_mb4[$i];
}
$gb18030_4_u_b[] = $begin_u;
$gb18030_4_u_e[] = $prev_u;
$gb18030_4_mb_e[] = $prev_mb;

// Gaps between blocks
$gb18030_4_u_mb_o[] = 0;
for ($i = 1, $cnt = count($gb18030_4_u_b); $i < $cnt; $i++) {
    $gb18030_4_u_mb_o[] = $gb18030_4_u_b[$i] - ($gb18030_4_u_e[$i - 1] + 1) + $gb18030_4_u_mb_o[count($gb18030_4_u_mb_o) - 1];
}

// Output GB 18030 tables

$u_us2 = $us2; // Keep original unsorted array
$u_mb2 = $mbs2;
array_multisort($u_us2, $u_mb2);
out_tabs($out, 'gb18030_2', $u_us2, $u_mb2, array(), true /*no_ind*/);

// Start codepoints `gb18030_4_u_b` array not needed by `zueci_u_gb18030()`
$cnt = count($gb18030_4_u_e);
$out[] = '';
$out[] = '/* End Unicode codepoints of blocks mapping consecutively to 4-byte multibyte blocks */';
$out[] = 'static const zueci_u16 zueci_gb18030_4_u_e[' . $cnt .'] = {';
out_tab_entries($out, $gb18030_4_u_e, $cnt);
$out[] = '};';
$cnt = count($gb18030_4_u_mb_o);
$out[] = '';
$out[] = '#ifndef ZUECI_EMBED_NO_TO_ECI';
$out[] = '/* Cumulative gaps between Unicode blocks mapping consecutively to 4-byte multibyte blocks,';
$out[] = '   used to adjust multibyte offsets */';
$out[] = 'static const zueci_u16 zueci_gb18030_4_u_mb_o[' . $cnt .'] = {';
out_tab_entries($out, $gb18030_4_u_mb_o, $cnt, true /*not_hex*/);
$out[] = '};';
$out[] = '#endif /* ZUECI_EMBED_NO_TO_ECI */';
$out[] = '';
$out[] = '#ifndef ZUECI_EMBED_NO_TO_UTF8';
$out[] = '/* End multibyte codepoints of blocks mapping consecutively to 4-byte multibyte blocks */';
$out[] = 'static const zueci_u32 zueci_gb18030_4_mb_e[' . $cnt .'] = {';
out_tab_entries($out, $gb18030_4_mb_e, $cnt);
$out[] = '};';
$out[] = '#endif /* ZUECI_EMBED_NO_TO_UTF8 */';

// Sort into multibyte order but remove former PUA entries first
for ($i = 0, $cnt = count($pua_change_2005_mb); $i < $cnt; $i++) {
    $j = array_search($pua_change_2005_mb[$i], $mbs2);
    assert($j !== false);
    array_splice($mbs2, $j, 1);
    array_splice($us2, $j, 1);
    assert(array_search($pua_change_2005_mb[$i], $mbs2) !== false); // Make sure non-PUA mapping exists
}
array_multisort($mbs2, $us2);
$cnt = count($mbs2);
$out[] = '';
$out[] = '#ifndef ZUECI_EMBED_NO_TO_UTF8';
$out[] = '/* Multibyte values sorted in multibyte order */';
$out[] = 'static const zueci_u16 zueci_gb18030_2_mb_mb[' . $cnt .'] = {';
out_tab_entries($out, $mbs2, $cnt);
$out[] = '};';
$out[] = '';
$out[] = '/* Unicode values sorted in multibyte order */';
$out[] = 'static const zueci_u16 zueci_gb18030_2_mb_u[' . $cnt .'] = {';
out_tab_entries($out, $us2, $cnt);
$out[] = '};';
$out[] = '#endif /* ZUECI_EMBED_NO_TO_UTF8 */';

$out[] = '';
$out[] = '#endif /* ZUECI_GB18030_H */';

file_put_contents($out_dirname . '/zueci_gb18030.h', implode("\n", $out) . "\n");

/* vim: set ts=4 sw=4 et : */
