From 93f22c78cde0e28365a0845e99ccd93cf706cbd3 Mon Sep 17 00:00:00 2001 From: Thomas Voss Date: Tue, 30 Apr 2024 23:05:37 +0200 Subject: Add even more 2-stage lookup tables --- gen/prop/bmg | 159 ++++++++++++++++++++++++++++++++++------------------ gen/prop/bpb | 157 +++++++++++++++++++++++++++++++++++----------------- gen/prop/bpt | 2 +- gen/prop/ccc | 2 +- gen/prop/ea | 166 +++++++++++++++++++++++++++++++++---------------------- gen/prop/equideo | 156 ++++++++++++++++++++++++++++++++++----------------- 6 files changed, 418 insertions(+), 224 deletions(-) (limited to 'gen') diff --git a/gen/prop/bmg b/gen/prop/bmg index 67124cd..61df16f 100755 --- a/gen/prop/bmg +++ b/gen/prop/bmg @@ -1,54 +1,105 @@ -#!/bin/sh - -set -e -cd "${0%/*}/../.." -exec >lib/unicode/prop/uprop_get_bmg.c - -gawk ' -BEGIN { - FS = " *(; *|#.*)" - - print "/* This file is autogenerated by gen/prop/bmg; DO NOT EDIT. */" - print "" - print "#include \"_bsearch.h\"" - print "#include \"macros.h\"" - print "#include \"rune.h\"" - print "#include \"unicode/prop.h\"" - print "" -} - -/^[^#]/ { - props[strtonum("0X" $1)] = strtonum("0X" $2) -} - -END { - print "static constexpr rune lookup_lat1[] = {" - for (i = 0; i < 0x100; i++) { - if (i % 8 == 0) - printf "\t" - printf "0x%02X,%s", (props[i] ? props[i] : 0), \ - i % 8 == 7 ? "\n" : " " - } - print "};" - print "" - print "static const struct {" - print "\trune k, v;" - print "} lookup[] = {" - - for (i = 0x100; i <= 0x10FFFF; i++) { - if (!props[i]) - continue - printf "\t{RUNE_C(0x%06X), RUNE_C(0x%06X)},\n", i, props[i] - } - - print "};" - print "" - print "_MLIB_DEFINE_BSEARCH_KV(rune, lookup, 0)" - print "" - print "rune" - print "uprop_get_bmg(rune ch)" - print "{" - print "\treturn ch < lengthof(lookup_lat1) ? lookup_lat1[ch] : mlib_lookup_kv(ch);" - print "}" -} -' data/BidiMirroring +#!/usr/bin/python3 + +import math + +from wcwidth import wcswidth + +from lib import * + + +longest = 0 + +def parse(file: str) -> list[bool]: + global longest + + xs = ['0'] * 0x110000 + with open(file, 'r') as f: + for line in f.readlines(): + if len(line.strip()) == 0 or line[0] == '#': + continue + + parts = line.split(';') + ranges = [int(x, 16) for x in parts[0].strip().split('..')] + prop = int(parts[1].split('#')[0].strip(), 16) + prop = ("U'" if prop > 0x7F else "'") + chr(prop) + "'" + longest = max(longest, wcswidth(prop)) + + for i in range(ranges[0], ranges[len(ranges) - 1] + 1): + xs[i] = prop + return xs + +def genfile(cs: list[tuple[bool, ...]], blksize: int) -> None: + Cs = cs + cs = list(dict.fromkeys(Cs)) + + print('''\ +/* This file is autogenerated by gen/prop/bmg; DO NOT EDIT. */ + +#include "unicode/prop.h" +''') + + print(f'static constexpr {typename(len(cs) - 1)} stage1[] = {{') + for i, c in enumerate(Cs): + print(f'%c%{len(str(len(cs) - 1))}d,' % ('\t' if i % 16 == 0 else ' ', cs.index(c)), end='') + if i % 16 == 15: + print() + print('};') + + print() + + ppc = columns(blksize, longest + 1) + print(f'static constexpr rune stage2[][{blksize}] = {{') + for c in cs: + for i in range(blksize // ppc): + print('\t{' if i == 0 else '\t ', end='') + for j in range(ppc): + print(c[i*ppc + j], end='') + if i < blksize // ppc - 1 or j < ppc - 1: + print(',', end='') + if j < ppc - 1: + print(' ' * (longest + 1 - wcswidth(c[i*ppc + j])), end='') + if i < blksize // ppc - 1: + print() + print('},') + print('};') + + print() + + print(f'''\ +rune +uprop_get_bmg(rune ch) +{{ + rune hc = stage2[stage1[ch / {blksize}]][ch % {blksize}]; + return hc == 0 ? ch : hc; +}}''') + +def main() -> None: + cwd_init() + xs = parse('data/BidiMirroring') + + blksize = -1 + smallest = math.inf + + for bs in powers_of_2(): + if bs > len(xs): + break + Cs = [tuple(x) for x in chunks(xs, bs)] + cs = set(Cs) + + sz_s1 = len(Cs) * isize(len(cs) - 1) + sz_s2 = len(cs) * bs * 4 + sz = sz_s1 + sz_s2 + + if sz < smallest: + smallest = sz + blksize = bs + + Cs = [tuple(x) for x in chunks(xs, blksize)] + with open('lib/unicode/prop/uprop_get_bmg.c', 'w') as f: + sys.stdout = f + genfile(Cs, blksize) + + report_size(len(xs), smallest) + +if __name__ == '__main__': + main() diff --git a/gen/prop/bpb b/gen/prop/bpb index 8c0b696..7cbaa66 100755 --- a/gen/prop/bpb +++ b/gen/prop/bpb @@ -1,52 +1,105 @@ -#!/bin/sh - -set -e -cd "${0%/*}/../.." -exec >lib/unicode/prop/uprop_get_bpb.c - -gawk ' -BEGIN { - FS = " *(; *|#.*)" - - print "/* This file is autogenerated by gen/prop/bpb; DO NOT EDIT. */" - print "" - print "#include \"_bsearch.h\"" - print "#include \"macros.h\"" - print "#include \"rune.h\"" - print "#include \"unicode/prop.h\"" - print "" -} - -/^[^#]/ { - props[strtonum("0X" $1)] = strtonum("0X" $2) -} - -END { - print "static constexpr rune lookup_lat1[] = {" - for (i = 0; i < 0x100; i++) { - if (i % 8 == 0) - printf "\t" - printf "0x%02X,%s", props[i] ? props[i] : 0, i % 8 == 7 ? "\n" : " " - } - print "};" - print "" - print "static const struct {" - print "\trune k, v;" - print "} lookup[] = {" - - for (i = 0x100; i <= 0x10FFFF; i++) { - if (props[i]) - printf "\t{RUNE_C(0x%06X), RUNE_C(0x%06X)},\n", i, props[i] - } - - print "};" - print "" - print "_MLIB_DEFINE_BSEARCH_KV(rune, lookup, 0)" - print "" - print "rune" - print "uprop_get_bpb(rune ch)" - print "{" - print "\treturn ch < lengthof(lookup_lat1) ? lookup_lat1[ch] : mlib_lookup_kv(ch);" - print "}" -} -' data/BidiBrackets +#!/usr/bin/python3 + +import math + +from wcwidth import wcswidth + +from lib import * + + +longest = 0 + +def parse(file: str) -> list[bool]: + global longest + + xs = ['0'] * 0x110000 + with open(file, 'r') as f: + for line in f.readlines(): + if len(line.strip()) == 0 or line[0] == '#': + continue + + parts = line.split(';') + ranges = [int(x, 16) for x in parts[0].strip().split('..')] + prop = int(parts[1].split('#')[0].strip(), 16) + prop = ("U'" if prop > 0x7F else "'") + chr(prop) + "'" + longest = max(longest, wcswidth(prop)) + + for i in range(ranges[0], ranges[len(ranges) - 1] + 1): + xs[i] = prop + return xs + +def genfile(cs: list[tuple[bool, ...]], blksize: int) -> None: + Cs = cs + cs = list(dict.fromkeys(Cs)) + + print('''\ +/* This file is autogenerated by gen/prop/bpb; DO NOT EDIT. */ + +#include "unicode/prop.h" +''') + + print(f'static constexpr {typename(len(cs) - 1)} stage1[] = {{') + for i, c in enumerate(Cs): + print(f'%c%{len(str(len(cs) - 1))}d,' % ('\t' if i % 16 == 0 else ' ', cs.index(c)), end='') + if i % 16 == 15: + print() + print('};') + + print() + + ppc = columns(blksize, longest + 1) + print(f'static constexpr rune stage2[][{blksize}] = {{') + for c in cs: + for i in range(blksize // ppc): + print('\t{' if i == 0 else '\t ', end='') + for j in range(ppc): + print(c[i*ppc + j], end='') + if i < blksize // ppc - 1 or j < ppc - 1: + print(',', end='') + if j < ppc - 1: + print(' ' * (longest + 1 - wcswidth(c[i*ppc + j])), end='') + if i < blksize // ppc - 1: + print() + print('},') + print('};') + + print() + + print(f'''\ +rune +uprop_get_bpb(rune ch) +{{ + rune hc = stage2[stage1[ch / {blksize}]][ch % {blksize}]; + return hc == 0 ? ch : hc; +}}''') + +def main() -> None: + cwd_init() + xs = parse('data/BidiBrackets') + + blksize = -1 + smallest = math.inf + + for bs in powers_of_2(): + if bs > len(xs): + break + Cs = [tuple(x) for x in chunks(xs, bs)] + cs = set(Cs) + + sz_s1 = len(Cs) * isize(len(cs) - 1) + sz_s2 = len(cs) * bs * 4 + sz = sz_s1 + sz_s2 + + if sz < smallest: + smallest = sz + blksize = bs + + Cs = [tuple(x) for x in chunks(xs, blksize)] + with open('lib/unicode/prop/uprop_get_bpb.c', 'w') as f: + sys.stdout = f + genfile(Cs, blksize) + + report_size(len(xs), smallest) + +if __name__ == '__main__': + main() diff --git a/gen/prop/bpt b/gen/prop/bpt index 72a9215..e2db55d 100755 --- a/gen/prop/bpt +++ b/gen/prop/bpt @@ -83,7 +83,7 @@ def main() -> None: cs = set(Cs) sz_s1 = len(Cs) * isize(len(cs) - 1) - sz_s2 = len(cs) * bs * 2 + sz_s2 = len(cs) * bs sz = sz_s1 + sz_s2 if sz < smallest: diff --git a/gen/prop/ccc b/gen/prop/ccc index 5339748..bc073ae 100755 --- a/gen/prop/ccc +++ b/gen/prop/ccc @@ -147,7 +147,7 @@ def main() -> None: cs = set(Cs) sz_s1 = len(Cs) * isize(len(cs) - 1) - sz_s2 = len(cs) * bs * 4 + sz_s2 = len(cs) * bs * 2 sz = sz_s1 + sz_s2 if sz < smallest: diff --git a/gen/prop/ea b/gen/prop/ea index 0d7f357..1e0211a 100755 --- a/gen/prop/ea +++ b/gen/prop/ea @@ -1,65 +1,101 @@ -#!/bin/sh - -set -e -cd "${0%/*}/../.." -exec >lib/unicode/prop/uprop_get_ea.c - -gawk ' -BEGIN { - FS = "( *#.*| *; +)" - - print "/* This file is autogenerated by gen/prop/ea; DO NOT EDIT. */" - print "" - print "#include \"_bsearch.h\"" - print "#include \"macros.h\"" - print "#include \"rune.h\"" - print "#include \"unicode/prop.h\"" - print "" -} - -/^[^#]/ { - n = split($1, a, /\.\./) - lo = strtonum("0X" a[1]) - hi = strtonum("0X" a[n]) - - for (i = lo; i <= hi; i++) { - gsub(/^; /, "", $2) - props[i] = "EA_" toupper($2) - } -} - -END { - print "static constexpr enum uprop_ea lookup_lat1[] = {" - for (i = 0; i < 0x100; i++) { - if (i % 8 == 0) - printf "\t" - printf "%-6s%s", (props[i] ? props[i] : "EA_NA") ",", \ - i % 8 == 7 ? "\n" : " " - } - print "};" - print "" - print "static const struct {" - print "\trune lo, hi;" - print "\tenum uprop_ea val;" - print "} lookup[] = {" - - for (i = 0x100; i <= 0x10FFFF; i++) { - if (!props[i]) - continue - lo = i - while (props[lo] == props[i + 1]) - i++ - printf "\t{RUNE_C(0x%06X), RUNE_C(0x%06X), %s},\n", lo, i, props[i] - } - - print "};" - print "" - print "_MLIB_DEFINE_BSEARCH(enum uprop_ea, lookup, EA_NA)" - print "" - print "enum uprop_ea" - print "uprop_get_ea(rune ch)" - print "{" - print "\treturn ch < lengthof(lookup_lat1) ? lookup_lat1[ch] : mlib_lookup(ch);" - print "}" -} -' data/DerivedEastAsianWidth | sed 's/\s*$//' +#!/usr/bin/python3 + +import math + +from lib import * + + +longest = 0 + +def parse(file: str) -> list[bool]: + global longest + + xs = ['EA_NA'] * 0x110000 + with open(file, 'r') as f: + for line in f.readlines(): + if len(line.strip()) == 0 or line[0] == '#': + continue + + parts = line.split(';') + ranges = [int(x, 16) for x in parts[0].strip().split('..')] + prop = 'EA_' + parts[1].split('#')[0].strip().upper() + longest = max(longest, len(prop)) + + for i in range(ranges[0], ranges[len(ranges) - 1] + 1): + xs[i] = prop + return xs + +def genfile(cs: list[tuple[bool, ...]], blksize: int) -> None: + Cs = cs + cs = list(dict.fromkeys(Cs)) + + print('''\ +/* This file is autogenerated by gen/prop/ea; DO NOT EDIT. */ + +#include "unicode/prop.h" +''') + + print(f'static constexpr {typename(len(cs) - 1)} stage1[] = {{') + for i, c in enumerate(Cs): + print(f'%c%{len(str(len(cs) - 1))}d,' % ('\t' if i % 16 == 0 else ' ', cs.index(c)), end='') + if i % 16 == 15: + print() + print('};') + + print() + + ppc = columns(blksize, longest + 1) + print(f'static constexpr enum uprop_ea stage2[][{blksize}] = {{') + for c in cs: + for i in range(blksize // ppc): + print('\t{' if i == 0 else '\t ', end='') + for j in range(ppc): + print(c[i*ppc + j], end='') + if i < blksize // ppc - 1 or j < ppc - 1: + print(',', end='') + if j < ppc - 1: + print(' ' * (longest + 1 - len(c[i*ppc + j])), end='') + if i < blksize // ppc - 1: + print() + print('},') + print('};') + + print() + + print(f'''\ +enum uprop_ea +uprop_get_ea(rune ch) +{{ + return stage2[stage1[ch / {blksize}]][ch % {blksize}]; +}}''') + +def main() -> None: + cwd_init() + xs = parse('data/DerivedEastAsianWidth') + + blksize = -1 + smallest = math.inf + + for bs in powers_of_2(): + if bs > len(xs): + break + Cs = [tuple(x) for x in chunks(xs, bs)] + cs = set(Cs) + + sz_s1 = len(Cs) * isize(len(cs) - 1) + sz_s2 = len(cs) * bs + sz = sz_s1 + sz_s2 + + if sz < smallest: + smallest = sz + blksize = bs + + Cs = [tuple(x) for x in chunks(xs, blksize)] + with open('lib/unicode/prop/uprop_get_ea.c', 'w') as f: + sys.stdout = f + genfile(Cs, blksize) + + report_size(len(xs), smallest) + +if __name__ == '__main__': + main() diff --git a/gen/prop/equideo b/gen/prop/equideo index 4f3ad2a..b1f5866 100755 --- a/gen/prop/equideo +++ b/gen/prop/equideo @@ -1,51 +1,105 @@ -#!/bin/sh - -set -e -cd "${0%/*}/../.." -exec >lib/unicode/prop/uprop_get_equideo.c - -gawk ' -BEGIN { - FS = "[ ;]+" - - print "/* This file is autogenerated by gen/prop/equideo; DO NOT EDIT. */" - print "" - print "#include \"_bsearch.h\"" - print "#include \"macros.h\"" - print "#include \"rune.h\"" - print "#include \"unicode/prop.h\"" - print "" -} - -/^[A-F0-9]{4}/ { - n = split($1, a, /\.\./) - lo = strtonum("0X" a[1]) - hi = strtonum("0X" a[n]) - - for (i = lo; i <= hi; i++) - props[i] = strtonum("0X" $2) -} - -END { - print "static const struct {" - print "\trune k, v;" - print "} lookup[] = {" - - for (i = 0; i <= 0x10FFFF; i++) { - if (props[i]) - printf "\t{RUNE_C(0x%06X), RUNE_C(0x%06X)},\n", i, props[i] - } - - print "};" - print "" - print "_MLIB_DEFINE_BSEARCH_KV(rune, lookup, \x27\\0\x27)" - print "" - print "rune" - print "uprop_get_equideo(rune ch)" - print "{" - print "\treturn ch < lookup[0].k || ch > lookup[lengthof(lookup) - 1].k" - print "\t\t? \x27\\0\x27" - print "\t\t: mlib_lookup_kv(ch);" - print "}" -} -' data/EquivalentUnifiedIdeograph +#!/usr/bin/python3 + +import math + +from wcwidth import wcswidth + +from lib import * + + +longest = 0 + +def parse(file: str) -> list[bool]: + global longest + + xs = ['0'] * 0x110000 + with open(file, 'r') as f: + for line in f.readlines(): + if len(line.strip()) == 0 or line[0] == '#': + continue + + parts = line.split(';') + ranges = [int(x, 16) for x in parts[0].strip().split('..')] + prop = int(parts[1].split('#')[0].strip(), 16) + prop = ("U'" if prop > 0x7F else "'") + chr(prop) + "'" + longest = max(longest, wcswidth(prop)) + + for i in range(ranges[0], ranges[len(ranges) - 1] + 1): + xs[i] = prop + return xs + +def genfile(cs: list[tuple[bool, ...]], blksize: int) -> None: + Cs = cs + cs = list(dict.fromkeys(Cs)) + + print('''\ +/* This file is autogenerated by gen/prop/equideo; DO NOT EDIT. */ + +#include "unicode/prop.h" +''') + + print(f'static constexpr {typename(len(cs) - 1)} stage1[] = {{') + for i, c in enumerate(Cs): + print(f'%c%{len(str(len(cs) - 1))}d,' % ('\t' if i % 16 == 0 else ' ', cs.index(c)), end='') + if i % 16 == 15: + print() + print('};') + + print() + + ppc = columns(blksize, longest + 1) + print(f'static constexpr rune stage2[][{blksize}] = {{') + for c in cs: + for i in range(blksize // ppc): + print('\t{' if i == 0 else '\t ', end='') + for j in range(ppc): + print(c[i*ppc + j], end='') + if i < blksize // ppc - 1 or j < ppc - 1: + print(',', end='') + if j < ppc - 1: + print(' ' * (longest + 1 - wcswidth(c[i*ppc + j])), end='') + if i < blksize // ppc - 1: + print() + print('},') + print('};') + + print() + + print(f'''\ +rune +uprop_get_equideo(rune ch) +{{ + rune hc = stage2[stage1[ch / {blksize}]][ch % {blksize}]; + return hc == 0 ? ch : hc; +}}''') + +def main() -> None: + cwd_init() + xs = parse('data/EquivalentUnifiedIdeograph') + + blksize = -1 + smallest = math.inf + + for bs in powers_of_2(): + if bs > len(xs): + break + Cs = [tuple(x) for x in chunks(xs, bs)] + cs = set(Cs) + + sz_s1 = len(Cs) * isize(len(cs) - 1) + sz_s2 = len(cs) * bs * 4 + sz = sz_s1 + sz_s2 + + if sz < smallest: + smallest = sz + blksize = bs + + Cs = [tuple(x) for x in chunks(xs, blksize)] + with open('lib/unicode/prop/uprop_get_equideo.c', 'w') as f: + sys.stdout = f + genfile(Cs, blksize) + + report_size(len(xs), smallest) + +if __name__ == '__main__': + main() -- cgit v1.2.3