summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore2
-rw-r--r--Cargo.lock994
-rw-r--r--Cargo.toml2
-rw-r--r--oryxc/.gitignore1
-rw-r--r--oryxc/Cargo.toml8
-rw-r--r--oryxc/build.rs289
-rw-r--r--oryxc/generated/line_terminator.rs13
-rw-r--r--oryxc/generated/pattern_white_space.rs13
-rw-r--r--oryxc/generated/xid_continue.rs13
-rw-r--r--oryxc/generated/xid_start.rs13
-rw-r--r--oryxc/src/unicode/line_terminator.rs136
-rw-r--r--oryxc/src/unicode/pattern_white_space.rs138
-rw-r--r--oryxc/src/unicode/xid_continue.rs1008
-rw-r--r--oryxc/src/unicode/xid_start.rs928
-rw-r--r--unigen/Cargo.toml7
-rwxr-xr-xunigen/fetch19
-rw-r--r--unigen/src/main.rs279
17 files changed, 1339 insertions, 2524 deletions
diff --git a/.gitignore b/.gitignore
index 72e2ff5..b42a62c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
target/
-unigen/data/
+data/
.idea/
diff --git a/Cargo.lock b/Cargo.lock
index b835f73..9c206b2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,6 +3,23 @@
version = 4
[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
name = "anstream"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -38,7 +55,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -49,28 +66,92 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
+name = "anyhow"
+version = "1.0.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
name = "boxcar"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e"
[[package]]
+name = "bumpalo"
+version = "3.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+
+[[package]]
+name = "bytes"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+
+[[package]]
+name = "bzip2"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
+dependencies = [
+ "libbz2-rs-sys",
+]
+
+[[package]]
+name = "cc"
+version = "1.2.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+]
+
+[[package]]
name = "clap"
version = "4.5.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -117,6 +198,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
+name = "constant_time_eq"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -142,6 +247,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -149,41 +264,243 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
- "hashbrown",
+ "hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
+name = "deflate64"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2"
+
+[[package]]
+name = "deranged"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "powerfmt",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 5.3.0",
+ "wasip2",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "r-efi 6.0.0",
+ "wasip2",
+ "wasip3",
+ "wasm-bindgen",
+]
+
+[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
+name = "hashbrown"
+version = "0.15.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+
+[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "http"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "id-arena"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
+
+[[package]]
+name = "indexmap"
+version = "2.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.16.1",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "inout"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
-name = "lexopt"
-version = "0.3.2"
+name = "itoa"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413"
+checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
+dependencies = [
+ "once_cell",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "leb128fmt"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
+
+[[package]]
+name = "libbz2-rs-sys"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
@@ -201,6 +518,43 @@ dependencies = [
]
[[package]]
+name = "log"
+version = "0.4.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+
+[[package]]
+name = "lzma-rust2"
+version = "0.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47bb1e988e6fb779cf720ad431242d3f03167c1b3f2b1aae7f1a94b2495b36ae"
+dependencies = [
+ "sha2",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
+
+[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -224,6 +578,8 @@ dependencies = [
"soa-rs",
"unicode-normalization",
"unicode-width",
+ "ureq",
+ "zip",
]
[[package]]
@@ -240,6 +596,22 @@ dependencies = [
]
[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
+[[package]]
name = "phf"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -283,6 +655,34 @@ dependencies = [
]
[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "ppmd-rust"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
+
+[[package]]
+name = "prettyplease"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
+[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -301,6 +701,18 @@ dependencies = [
]
[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -310,12 +722,73 @@ dependencies = [
]
[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
+dependencies = [
+ "log",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
+[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -345,6 +818,53 @@ dependencies = [
]
[[package]]
+name = "serde_json"
+version = "1.0.149"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "sha1"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+
+[[package]]
name = "siphasher"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -383,6 +903,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
+[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -394,6 +920,26 @@ dependencies = [
]
[[package]]
+name = "time"
+version = "0.3.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
+dependencies = [
+ "deranged",
+ "js-sys",
+ "num-conv",
+ "powerfmt",
+ "serde_core",
+ "time-core",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
+
+[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -409,6 +955,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
+name = "typed-path"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
+
+[[package]]
+name = "typenum"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+
+[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -430,19 +988,177 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
-name = "unigen"
-version = "0.1.0"
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "ureq"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc"
dependencies = [
- "lexopt",
+ "base64",
+ "flate2",
+ "log",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf-8",
+ "webpki-roots",
]
[[package]]
+name = "ureq-proto"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
+dependencies = [
+ "base64",
+ "http",
+ "httparse",
+ "log",
+]
+
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
+[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasip2"
+version = "1.0.2+wasi-0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasip3"
+version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "wasm-encoder"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
+dependencies = [
+ "leb128fmt",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasm-metadata"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
+dependencies = [
+ "anyhow",
+ "indexmap",
+ "wasm-encoder",
+ "wasmparser",
+]
+
+[[package]]
+name = "wasmparser"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
+dependencies = [
+ "bitflags",
+ "hashbrown 0.15.5",
+ "indexmap",
+ "semver",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -450,9 +1166,269 @@ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
+dependencies = [
+ "wit-bindgen-rust-macro",
+]
+
+[[package]]
+name = "wit-bindgen-core"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
+dependencies = [
+ "anyhow",
+ "heck",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-bindgen-rust"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
+dependencies = [
+ "anyhow",
+ "heck",
+ "indexmap",
+ "prettyplease",
+ "syn",
+ "wasm-metadata",
+ "wit-bindgen-core",
+ "wit-component",
+]
+
+[[package]]
+name = "wit-bindgen-rust-macro"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
+dependencies = [
+ "anyhow",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wit-bindgen-core",
+ "wit-bindgen-rust",
+]
+
+[[package]]
+name = "wit-component"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
+dependencies = [
+ "anyhow",
+ "bitflags",
+ "indexmap",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "wasm-encoder",
+ "wasm-metadata",
+ "wasmparser",
+ "wit-parser",
+]
+
+[[package]]
+name = "wit-parser"
+version = "0.244.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
+dependencies = [
+ "anyhow",
+ "id-arena",
+ "indexmap",
+ "log",
+ "semver",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "unicode-xid",
+ "wasmparser",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "zip"
+version = "8.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004"
+dependencies = [
+ "aes",
+ "bzip2",
+ "constant_time_eq",
+ "crc32fast",
+ "deflate64",
+ "flate2",
+ "getrandom 0.4.2",
+ "hmac",
+ "indexmap",
+ "lzma-rust2",
+ "memchr",
+ "pbkdf2",
+ "ppmd-rust",
+ "sha1",
+ "time",
+ "typed-path",
+ "zeroize",
+ "zopfli",
+ "zstd",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+
+[[package]]
+name = "zopfli"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "log",
+ "simd-adler32",
+]
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
diff --git a/Cargo.toml b/Cargo.toml
index cbe097e..4221c83 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,4 @@
[workspace]
resolver = "3"
default-members = ["oryxc"]
-members = ["oryxc", "unigen"]
+members = ["oryxc"]
diff --git a/oryxc/.gitignore b/oryxc/.gitignore
deleted file mode 100644
index ea8c4bf..0000000
--- a/oryxc/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target
diff --git a/oryxc/Cargo.toml b/oryxc/Cargo.toml
index a7ab1e4..4230086 100644
--- a/oryxc/Cargo.toml
+++ b/oryxc/Cargo.toml
@@ -8,8 +8,14 @@ boxcar = "0.2.14"
clap = { version = "4", features = ["derive"] }
crossbeam-deque = "0.8.6"
dashmap = "6.1.0"
-# num-rational = "0.4.2"
phf = { version = "0.13.1", features = ["macros"] }
soa-rs = "0.9.1"
unicode-normalization = "0.1.25"
unicode-width = "0.2.2"
+
+[features]
+fetch = ["dep:ureq", "dep:zip"]
+
+[build-dependencies]
+ureq = { version = "3.2.0", optional = true }
+zip = { version = "8.2.0", optional = true }
diff --git a/oryxc/build.rs b/oryxc/build.rs
new file mode 100644
index 0000000..8de0327
--- /dev/null
+++ b/oryxc/build.rs
@@ -0,0 +1,289 @@
+const NAMES: &[&str] = &[
+ "xid_start",
+ "xid_continue",
+ "pattern_white_space",
+ "line_terminator",
+];
+
+fn main() {
+ use std::env;
+
+ let out_dir = env::var("OUT_DIR").unwrap();
+ let root = env::var("CARGO_MANIFEST_DIR").unwrap();
+ let generated = format!("{root}/generated");
+
+ #[cfg(feature = "fetch")]
+ fetch::run(&out_dir, &generated);
+
+ #[cfg(not(feature = "fetch"))]
+ fallback::run(&out_dir, &generated);
+}
+
+#[cfg(feature = "fetch")]
+mod fetch {
+ use std::collections::HashMap;
+ use std::fs;
+ use std::io::{
+ self,
+ BufRead,
+ BufReader,
+ Cursor,
+ Read,
+ Write,
+ };
+
+ const MIN_SHIFT: usize = 1;
+ const MAX_SHIFT: usize = 22;
+ const UCD_URL: &str =
+ "https://www.unicode.org/Public/zipped/latest/UCD.zip";
+
+ pub fn run(out_dir: &str, generated: &str) {
+ let data = format!("{out_dir}/data");
+ let derived = format!("{data}/DerivedCoreProperties.txt");
+ let proplist = format!("{data}/PropList.txt");
+
+ println!("cargo:rerun-if-changed={derived}");
+ println!("cargo:rerun-if-changed={proplist}");
+
+ if !fs::exists(&derived).unwrap_or(false)
+ || !fs::exists(&proplist).unwrap_or(false)
+ {
+ let mut bytes = Vec::new();
+ ureq::get(UCD_URL)
+ .call()
+ .expect("failed to download UCD.zip")
+ .into_reader()
+ .read_to_end(&mut bytes)
+ .expect("failed to read UCD.zip");
+
+ fs::create_dir_all(&data).unwrap();
+ zip::ZipArchive::new(Cursor::new(bytes))
+ .expect("failed to open UCD.zip")
+ .extract(&data)
+ .expect("failed to extract UCD.zip");
+
+ // XID_Start and XID_Continue additions
+ let mut f = fs::OpenOptions::new()
+ .append(true)
+ .open(&derived)
+ .expect("failed to open DerivedCoreProperties.txt");
+ writeln!(
+ f,
+ "0024 ; XID_Start # Pc DOLLAR SIGN\n\
+ 005F ; XID_Start # Pc LOW LINE\n\
+ 2032..2034 ; XID_Continue # Po [3] PRIME..TRIPLE PRIME\n\
+ 2057 ; XID_Continue # Po QUADRUPLE PRIME"
+ )
+ .unwrap();
+ }
+
+ generate_from_file(out_dir, &derived, "XID_Start", "xid_start");
+ generate_from_file(out_dir, &derived, "XID_Continue", "xid_continue");
+ generate_from_file(
+ out_dir,
+ &proplist,
+ "Pattern_White_Space",
+ "pattern_white_space",
+ );
+ generate_from_codepoints(
+ out_dir,
+ &[
+ '\u{A}', '\u{B}', '\u{C}', '\u{D}', '\u{85}', '\u{2028}',
+ '\u{2029}',
+ ],
+ "line_terminator",
+ );
+
+ // Keep generated/ in sync so it can be committed as a fallback
+ fs::create_dir_all(generated).unwrap();
+ for name in super::NAMES {
+ fs::copy(
+ format!("{out_dir}/{name}.rs"),
+ format!("{generated}/{name}.rs"),
+ )
+ .unwrap_or_else(|e| {
+ panic!("failed to copy {name}.rs to generated/: {e}")
+ });
+ }
+ }
+
+ fn generate_from_file(out_dir: &str, path: &str, prop: &str, name: &str) {
+ let mut bitmap = vec![false; 0x110000];
+ parse_file(path, prop, &mut bitmap)
+ .unwrap_or_else(|e| panic!("failed to read {path}: {e}"));
+ write_output(out_dir, name, &bitmap);
+ }
+
+ fn generate_from_codepoints(
+ out_dir: &str,
+ codepoints: &[char],
+ name: &str,
+ ) {
+ let mut bitmap = vec![false; 0x110000];
+ for &c in codepoints {
+ bitmap[c as usize] = true;
+ }
+ write_output(out_dir, name, &bitmap);
+ }
+
+ fn write_output(out_dir: &str, name: &str, bitmap: &[bool]) {
+ let (shift, lvl1, lvl2) = optimize_tables(bitmap);
+ let mut f = fs::File::create(format!("{out_dir}/{name}.rs")).unwrap();
+ generate_code(&mut f, name, shift, &lvl1, &lvl2);
+ }
+
+ fn optimize_tables(bitmap: &[bool]) -> (usize, Vec<u16>, Vec<u64>) {
+ let mut minsz = usize::MAX;
+ let mut config = (0, Vec::new(), Vec::new());
+
+ for i in MIN_SHIFT..=MAX_SHIFT {
+ let (l1, l2) = build_tables(bitmap, i);
+ let sz = l1.len() * 2 + l2.len() * 8;
+ if sz < minsz {
+ minsz = sz;
+ config = (i, l1, l2);
+ }
+ }
+
+ config
+ }
+
+ fn parse_file(
+ path: &str,
+ prop: &str,
+ bitmap: &mut [bool],
+ ) -> io::Result<()> {
+ let file = fs::File::open(path)?;
+ let reader = BufReader::new(file);
+
+ for line in reader.lines() {
+ let line = line?;
+ let line = line.split('#').next().unwrap_or("").trim();
+ if line.is_empty() {
+ continue;
+ }
+
+ let parts: Vec<&str> = line.split(';').map(|s| s.trim()).collect();
+ if parts.len() < 2 || parts[1] != prop {
+ continue;
+ }
+
+ let (beg, end) = if parts[0].contains("..") {
+ let mut range = parts[0].split("..");
+ (
+ u32::from_str_radix(range.next().unwrap(), 16).unwrap(),
+ u32::from_str_radix(range.next().unwrap(), 16).unwrap(),
+ )
+ } else {
+ let val = u32::from_str_radix(parts[0], 16).unwrap();
+ (val, val)
+ };
+
+ for cp in beg..=end {
+ if (cp as usize) < bitmap.len() {
+ bitmap[cp as usize] = true;
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ fn build_tables(bitmap: &[bool], shift: usize) -> (Vec<u16>, Vec<u64>) {
+ let blksz = 1 << shift;
+ let u64s_per_block = (blksz + 63) / 64;
+
+ let mut lvl2: Vec<u64> = Vec::new();
+ let mut lvl1: Vec<u16> = Vec::new();
+ let mut blkmap: HashMap<Vec<u64>, u16> = HashMap::new();
+
+ for chunk in bitmap.chunks(blksz) {
+ let mut blkdata = vec![0u64; u64s_per_block];
+
+ for (i, &bit) in chunk.iter().enumerate() {
+ if bit {
+ let word_idx = i / 64;
+ let bit_idx = i % 64;
+ blkdata[word_idx] |= 1 << bit_idx;
+ }
+ }
+
+ if let Some(&i) = blkmap.get(&blkdata) {
+ lvl1.push(i);
+ } else {
+ let i = (lvl2.len() / u64s_per_block) as u16;
+ lvl2.extend_from_slice(&blkdata);
+ blkmap.insert(blkdata, i);
+ lvl1.push(i);
+ }
+ }
+
+ (lvl1, lvl2)
+ }
+
+ fn generate_code(
+ f: &mut impl Write,
+ prop_name: &str,
+ shift: usize,
+ level1: &[u16],
+ level2: &[u64],
+ ) {
+ let upper_name = prop_name.to_uppercase();
+ let lower_name = prop_name.to_lowercase();
+ let block_size = 1 << shift;
+ let mask = block_size - 1;
+ let u64s_per_block = (block_size + 63) / 64;
+
+ let pred_name = if lower_name.contains('_') {
+ format!("{lower_name}_p")
+ } else {
+ format!("{lower_name}p")
+ };
+
+ writeln!(f, "/* Autogenerated – DO NOT EDIT */").unwrap();
+ writeln!(f).unwrap();
+ writeln!(
+ f,
+ "static {upper_name}_L1: [u16; {}] = {level1:?};",
+ level1.len()
+ )
+ .unwrap();
+ writeln!(
+ f,
+ "static {upper_name}_L2: [u64; {}] = {level2:?};",
+ level2.len()
+ )
+ .unwrap();
+ writeln!(f, "#[inline]").unwrap();
+ writeln!(f, "pub fn {pred_name}(c: char) -> bool {{").unwrap();
+ writeln!(f, "\tlet cp = c as usize;").unwrap();
+ writeln!(f, "\tlet blki = unsafe {{ *{upper_name}_L1.get_unchecked(cp >> {shift}) }} as usize;").unwrap();
+ writeln!(f, "\tlet in_blk_offset_p = cp & 0x{mask:X};").unwrap();
+ if u64s_per_block == 1 {
+ writeln!(f, "\tunsafe {{ return ({upper_name}_L2.get_unchecked(blki) & (1 << in_blk_offset_p)) != 0; }}").unwrap();
+ } else {
+ writeln!(
+ f,
+ "\tlet wordi = (blki * {u64s_per_block}) + (in_blk_offset_p >> 6);"
+ )
+ .unwrap();
+ writeln!(f, "\tlet biti = in_blk_offset_p & 0x3F;").unwrap();
+ writeln!(f, "\tunsafe {{ return (*{upper_name}_L2.get_unchecked(wordi) & (1 << biti)) != 0; }}").unwrap();
+ }
+ writeln!(f, "}}").unwrap();
+ }
+}
+
+#[cfg(not(feature = "fetch"))]
+mod fallback {
+ use std::fs;
+
+ pub fn run(out_dir: &str, generated: &str) {
+ for name in super::NAMES {
+ let src = format!("{generated}/{name}.rs");
+ println!("cargo:rerun-if-changed={src}");
+ fs::copy(&src, format!("{out_dir}/{name}.rs"))
+ .unwrap_or_else(|e| panic!("failed to copy {src}: {e}"));
+ }
+ }
+}
diff --git a/oryxc/generated/line_terminator.rs b/oryxc/generated/line_terminator.rs
new file mode 100644
index 0000000..d29f4bd
--- /dev/null
+++ b/oryxc/generated/line_terminator.rs
@@ -0,0 +1,13 @@
+/* Autogenerated – DO NOT EDIT */
+
+static LINE_TERMINATOR_L1: [u16; 544] = [0, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
+static LINE_TERMINATOR_L2: [u64; 96] = [15360, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298534883328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+#[inline]
+pub fn line_terminator_p(c: char) -> bool {
+ let cp = c as usize;
+ let blki = unsafe { *LINE_TERMINATOR_L1.get_unchecked(cp >> 11) } as usize;
+ let in_blk_offset_p = cp & 0x7FF;
+ let wordi = (blki * 32) + (in_blk_offset_p >> 6);
+ let biti = in_blk_offset_p & 0x3F;
+ unsafe { return (*LINE_TERMINATOR_L2.get_unchecked(wordi) & (1 << biti)) != 0; }
+}
diff --git a/oryxc/generated/pattern_white_space.rs b/oryxc/generated/pattern_white_space.rs
new file mode 100644
index 0000000..e71b59d
--- /dev/null
+++ b/oryxc/generated/pattern_white_space.rs
@@ -0,0 +1,13 @@
+/* Autogenerated – DO NOT EDIT */
+
+static PATTERN_WHITE_SPACE_L1: [u16; 544] = [0, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
+static PATTERN_WHITE_SPACE_L2: [u64; 96] = [4294983168, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298534932480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+#[inline]
+pub fn pattern_white_space_p(c: char) -> bool {
+ let cp = c as usize;
+ let blki = unsafe { *PATTERN_WHITE_SPACE_L1.get_unchecked(cp >> 11) } as usize;
+ let in_blk_offset_p = cp & 0x7FF;
+ let wordi = (blki * 32) + (in_blk_offset_p >> 6);
+ let biti = in_blk_offset_p & 0x3F;
+ unsafe { return (*PATTERN_WHITE_SPACE_L2.get_unchecked(wordi) & (1 << biti)) != 0; }
+}
diff --git a/oryxc/generated/xid_continue.rs b/oryxc/generated/xid_continue.rs
new file mode 100644
index 0000000..834672c
--- /dev/null
+++ b/oryxc/generated/xid_continue.rs
@@ -0,0 +1,13 @@
+/* Autogenerated – DO NOT EDIT */
+
+static XID_CONTINUE_L1: [u16; 1088] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 12, 12, 12, 12, 12, 12, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 14, 15, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 16, 9, 9, 9, 9, 9, 9, 9, 9, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 9, 29, 12, 30, 12, 12, 31, 32, 9, 9, 9, 9, 9, 9, 33, 9, 34, 35, 12, 12, 12, 12, 12, 12, 12, 36, 9, 9, 9, 9, 9, 9, 9, 37, 38, 9, 9, 39, 9, 9, 9, 40, 41, 42, 43, 44, 45, 46, 47, 48, 9, 9, 49, 9, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 50, 12, 12, 12, 12, 51, 12, 12, 12, 12, 52, 12, 12, 12, 12, 12, 12, 53, 54, 9, 9, 55, 9, 12, 12, 12, 12, 56, 12, 12, 12, 12, 12, 12, 12, 12, 57, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 58, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9];
+static XID_CONTINUE_L2: [u64; 944] = [287948901175001088, 576460745995190270, 333270770471927808, 18410715276682199039, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 88094074470339, 18446744073709551615, 13321647697761927167, 18446744056529672128, 18428729675200069631, 18446744073709551615, 18446744073709551615, 18446744073709550843, 18446744073709551615, 18446462598732840959, 18446744069456527359, 13835058055282033151, 2119858418286774, 18446744069548736512, 18446678103011885055, 18446744073709551615, 11529212845433552895, 18446744073709486080, 18446744073709545471, 1125899906842623, 2612087783874887679, 70368744177663, 18446471390799331327, 18446744073701228287, 18446744056529682431, 18446744073709551615, 18446462392574410751, 17565725197581524975, 5765733215448889759, 15235112390417287150, 18014125208779143, 17576984196650090478, 18302910150157089727, 17576984196649951214, 844217444219295, 14123225865944680428, 281200107273671, 17582050746231021567, 281265452367327, 17577547146603651055, 4221916082617823, 18446744073709412351, 18158794964244397535, 3457638613854978030, 3658904103781503, 576460752303423486, 67076095, 4611685674830002134, 4093607775, 14024213633433600001, 18446216308128218879, 2305843009196916703, 64, 18446744073709551615, 18446744073709487103, 18446744070488326143, 17870283321406070975, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070446333439, 9168765891372858879, 18446744073701162813, 18446744073696837631, 1123704775901183, 18446744069414649855, 4557642822898941951, 18446744073709551614, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446638520593285119, 18446744069548802046, 144053615424700415, 9007197111451647, 3905461007941631, 18446744073709551615, 4394566287359, 18446744069481674752, 144115188075855871, 18446471394825863167, 18014398509481983, 1152657619668697087, 8796093022207936, 18446480190918885375, 134153215, 18446744069683019775, 11529215043920986111, 13834777130128311295, 17588964818943, 18446744073709551615, 4494803601399807, 18446744073709551615, 4503599627370495, 72057594037927935, 4611686018427380735, 16717361816799217663, 576460752302833664, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070475743231, 4611686017001275199, 6908521828386340863, 2295745090394464220, 9231253336202686464, 9223934986817634305, 536805376, 562821641207808, 17582049991377026180, 18446744069414601696, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 4494940973301759, 18446498607738650623, 9223513873854758911, 9187201948305063935, 18446744071553646463, 0, 0, 0, 0, 0, 0, 0, 0, 2251518330118602976, 18446744073709551614, 18446744068986765311, 18446744073709551615, 18446462598732840928, 18446744073709551615, 18446744069414617087, 18446462598732840960, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 8191, 4611686018427322368, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 17592185987071, 13830835930631503871, 18446744073709551615, 1125899906842623, 18446744060816261120, 18446744073709551615, 18446744073709550079, 18446181124293001215, 18691697672191, 4503599627370495, 18446744073709551615, 16789419406609285183, 18446532967477018623, 2305843004919775231, 18446744073709551615, 9223372032626884609, 36028797018963967, 18194542490348896255, 18446744073709551615, 35184368733388807, 18446602782178705022, 18446466996645134335, 18446744073709551615, 288010473826156543, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446462667452317695, 1152921504606845055, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446532967477018623, 18446744073709551615, 67108863, 6881498031078244479, 18446744073709551579, 1125899906842623, 18446744073709027328, 18446744073709551615, 18446744006063816703, 18446744073709551615, 18446744073709551615, 4611686018427387903, 18446744073709486080, 18446744073709355007, 287948901175001343, 7036870122864639, 12288634533233876992, 18446744073709551615, 2305843009213693951, 9799832780635308032, 18446743936404815870, 9223372036854775807, 486341884, 13258596753222922239, 1073692671, 18446744073709551615, 576460752303423487, 0, 9007199254740991, 0, 2305843009213693952, 0, 0, 18446744069951455231, 4295098367, 18446708893632430079, 576460752303359999, 18446744070488326143, 4128527, 18446744073709551615, 18446744073709551615, 18446466993558126591, 1152921504591118335, 18446463698244468735, 17870001915148894207, 2016486715970549759, 4503599627370495, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 36028797018963967, 1095220854783, 575897802350002111, 0, 10502394331027995967, 36028792728190975, 2147483647, 15762594400829440, 288230371860938751, 67108863, 13907115649320091647, 0, 9745789593611923567, 2305843004918726656, 536870911, 549755813631, 18014398509481983, 2251795522912255, 262143, 0, 18446744073709551615, 511, 2251799813685247, 2251799813685247, 287950000686628863, 18446671780820025343, 63, 0, 0, 0, 875211255709695, 18158513697557840124, 18446463149025525759, 18446462598732972031, 18446462598732841023, 36028792723996703, 18446744073709551615, 9241386160486350975, 576460752303423487, 287951100198191108, 18437736874454810623, 22517998136787184, 18446744073709551615, 402644511, 13907115649319829503, 3, 18446464796682337663, 287957697268023295, 18153444948953374703, 8760701963286943, 18428729675200023551, 25770850213, 18446744073709551615, 16173172735, 18446744073709551615, 67043519, 0, 0, 18392700878181105663, 1056964609, 18446744073709551615, 67043345, 144115188075855871, 68719412223, 287966492958392319, 127, 0, 0, 576460752303423487, 0, 18446744069414584320, 9223376434901286911, 17996384110963061375, 67043343, 18446740770879700992, 120208752639, 9223372036854775807, 18446744073709486208, 18446462599336820735, 144115188075855871, 0, 1095216660480, 0, 287948909764935679, 18410715276690587135, 18445618173869752321, 36027697507139583, 0, 13006395723845991295, 18446741595580465407, 18446466992517644287, 4394019979263, 0, 0, 0, 36028792723996672, 14411518807585456127, 134152199, 281474976710656, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 67108863, 0, 18446744073709551615, 140737488355327, 18446744073709551615, 18446744073709551615, 18446744073709551615, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709486080, 562949953421311, 281474976710655, 18446744069418778623, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 576460752303423487, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288230376151711743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 144115188075855871, 18446466994631868415, 9223372036854775807, 8796093022143487, 36028797018963967, 16212958624241090575, 65535, 0, 0, 0, 0, 0, 0, 287984085547089919, 0, 0, 0, 18446744073709551615, 18014398505187016704, 1048575, 18446744073709551615, 18446744073709520895, 4294934783, 35747438006370304, 18446744073709551615, 18446744073709551615, 18446744073709551615, 9223372036858970111, 2147483647, 0, 18446744073709551615, 2251799813685247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065665457643847680, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1125934266580991, 18446463629527547904, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1152921504606846975, 0, 0, 0, 0, 18446744073709551615, 2305570330330005503, 1677656575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287948901175001088, 0, 0, 0, 0, 0, 0, 0, 0, 18446532967477018623, 127, 0, 0, 0, 0, 0, 0, 0, 17872504197455282176, 65970697670631, 0, 0, 28, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073707454463, 17005555242810474495, 18446744073709551599, 8935141660164089791, 18446744073709419615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446743249075830783, 17870283321271910397, 18437736874452713471, 18446603336221163519, 18446741874686295551, 18446744073709539319, 0, 0, 0, 0, 0, 0, 0, 0, 17906312118425092095, 9042383626829823, 281470547525648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8660801552383, 0, 0, 0, 18446471240106377087, 70368744177663, 32768, 0, 4611439727822766079, 17407, 0, 0, 0, 0, 140737488289792, 288230376151711743, 0, 0, 0, 0, 0, 0, 0, 288230376151646208, 0, 0, 0, 576460752303357952, 0, 0, 0, 13853072451644162047, 0, 0, 0, 9223213153129594880, 18446744073709551615, 18446744073709551615, 18446744073709551615, 8323103, 18446744073709551615, 67047423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790380184120328175, 6843210385291930244, 1152917029519358975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287948901175001088, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 4294967295, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070488326143, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446532967477018623, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446462607322775551, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1073741823, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1073741823, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709488127, 18446744073709551615, 18446744073709551615, 18446744073709551615, 288230376151711743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 281474976710655, 0, 0, 0, 0, 0, 0, 0, 0];
+#[inline]
+pub fn xid_continue_p(c: char) -> bool {
+ let cp = c as usize;
+ let blki = unsafe { *XID_CONTINUE_L1.get_unchecked(cp >> 10) } as usize;
+ let in_blk_offset_p = cp & 0x3FF;
+ let wordi = (blki * 16) + (in_blk_offset_p >> 6);
+ let biti = in_blk_offset_p & 0x3F;
+ unsafe { return (*XID_CONTINUE_L2.get_unchecked(wordi) & (1 << biti)) != 0; }
+}
diff --git a/oryxc/generated/xid_start.rs b/oryxc/generated/xid_start.rs
new file mode 100644
index 0000000..78b5844
--- /dev/null
+++ b/oryxc/generated/xid_start.rs
@@ -0,0 +1,13 @@
+/* Autogenerated – DO NOT EDIT */
+
+static XID_START_L1: [u16; 1088] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 12, 12, 12, 12, 12, 12, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 14, 15, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 16, 9, 9, 9, 9, 9, 9, 9, 9, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 9, 29, 12, 30, 12, 12, 31, 32, 9, 9, 9, 9, 9, 9, 33, 9, 34, 35, 12, 12, 12, 12, 12, 12, 12, 36, 9, 9, 9, 9, 9, 9, 9, 37, 38, 9, 9, 39, 9, 9, 9, 9, 9, 40, 9, 41, 42, 43, 44, 45, 9, 9, 9, 9, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 46, 12, 12, 12, 12, 47, 12, 12, 12, 12, 48, 12, 12, 12, 12, 12, 12, 49, 50, 9, 9, 51, 9, 12, 12, 12, 12, 52, 12, 12, 12, 12, 12, 12, 12, 12, 53, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9];
+static XID_START_L2: [u64; 864] = [68719476736, 576460745995190270, 297241973452963840, 18410715276682199039, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 88094074470339, 0, 13321366222785216512, 18446744056529672000, 18428729675200069631, 18446744073709551615, 18446744073709551615, 18446744073709550595, 18446744073709551615, 18446462598732840959, 18446744069456527359, 511, 2119858418286592, 18446744069414584320, 18446392229988665343, 18446744073709551615, 11241196188469297151, 281474976514048, 18446744073709543424, 563224831328255, 301749971126844416, 1168302407679, 18446471390564450303, 18446744069414649599, 1023, 2594073385365405680, 18446181140919287808, 2577745637692514273, 1153765945374687232, 247132830528276448, 7881300924956672, 2589004636761079776, 144115200960823296, 2589004636760940512, 562965791113216, 288167810662516712, 65536, 2594071186342010848, 13807648768, 2589567586714640353, 1688864624214016, 2882303761516978160, 18158513712597581824, 3457638613854978016, 127, 1688849860263934, 127, 2307531515476572118, 4026531935, 1, 35184372088575, 7936, 0, 9223380832947798015, 18438229877581611008, 18446744069414600707, 17870283321406070975, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070446333439, 9168765891372858879, 18446744073701162813, 18446744073696837631, 134217727, 18446744069414649855, 4557642822898941951, 18446744073709551614, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446638520593285119, 18446744069548802046, 144053615424700415, 1125897759621119, 527761286627327, 4503599627370495, 276824064, 18446744069414584320, 144115188075855871, 18446469195802607615, 18014398509481983, 2147483647, 8796093022142464, 18446480190918885375, 1023, 18446744069422972927, 2097151, 549755813888, 0, 4503599627370464, 8160, 18158724812380307448, 274877906943, 68719476735, 4611686018360336384, 16717361816799217663, 319718190147960832, 18446744073709551615, 18446744073709551615, 18446744073709551615, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070475743231, 4611686017001275199, 6908521828386340863, 2295745090394464220, 0, 9223934986808197120, 536805376, 0, 17582049991377026180, 18446744069414601696, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 3509778554814463, 18446498607738650623, 141836999983103, 9187201948305063935, 2139062143, 0, 0, 0, 0, 0, 0, 0, 0, 2251241253188403424, 18446744073709551614, 18446744068886102015, 17870283321406128127, 18446462598732840928, 18446744073709551615, 18446744069414617087, 18446462598732840960, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 8191, 4611686018427322368, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 13198434443263, 9223512774343131135, 18446744070488326143, 281474976710655, 18446744060816261120, 18446744073709551615, 18446744073709550079, 18446181124293001215, 34359736251, 4503599627370495, 4503599627370492, 7564921474075590656, 18446462873610746880, 2305843004918726783, 2251799813685232, 8935422993945886720, 2199023255551, 14159317224157876215, 4495436853045886975, 7890092085477381, 18446602782178705022, 18446466996645134335, 18446744073709551615, 34359738367, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446462667452317695, 1152921504606845055, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446532967477018623, 18446744073709551615, 67108863, 6881498030004502655, 18446744073709551579, 1125899906842623, 18446744073709027328, 18446744073709551615, 18446744006063816703, 18446744073709551615, 18446744073709551615, 4611686018427387903, 18446744073709486080, 18446744073709355007, 287948901175001343, 0, 12288634533233819648, 18446744073709551615, 2305843009213693951, 576460743713488896, 18446743798965862398, 9223372033633550335, 486341884, 13258596753222922239, 1073692671, 18446744073709551615, 576460752303423487, 0, 9007199254740991, 0, 0, 0, 0, 18446744069951455231, 131071, 18446708893632430079, 18014398509418495, 18446744070488326143, 4128527, 18446744073709551615, 18446744073709551615, 18446462599806582783, 1152921504591118335, 18446463698244468735, 17870001915148894207, 2016486715970549759, 4503599627370495, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 36028797018963967, 1095220854783, 575897802350002111, 0, 10502394331027995967, 36028792728190975, 2147483647, 15762594400829440, 288230371860938751, 67108863, 13907115649320091647, 0, 18014398491590657, 2305843004918726656, 536870911, 137438953215, 18014398509481983, 2251795522912255, 262143, 0, 18446744073709551615, 511, 2251799813685247, 2251799813685247, 68719476735, 18446603611099102208, 63, 0, 0, 0, 848822976643071, 252, 18446463149025525759, 18446462598732841023, 18446462598732840963, 36028792723996703, 72057594037927928, 10696049115004928, 281474976710648, 2199023190016, 549755813880, 20266198323101840, 2251799813685240, 335544350, 9223389629040558079, 1, 18446464796682337663, 2147483647, 2589004636760940512, 16643063808, 54043195528399871, 655360, 9007199254740991, 15032387456, 281474976710655, 176, 0, 0, 140737488355327, 251658240, 281474976710655, 16, 72066390130950143, 0, 134217727, 127, 0, 0, 17592186044415, 0, 18446744069414584320, 9223372041149743103, 9223653511822045823, 2, 18446740770879700992, 42949804031, 290482175965394945, 18446744073441181696, 18446462599269712895, 144115188075855871, 0, 0, 0, 8589934591, 140737488354815, 18445618173802708993, 65535, 0, 562949953420159, 18446741595513421888, 18446462598749619199, 268435455, 0, 0, 0, 2251795518717952, 4503599627239412, 0, 281474976710656, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 67108863, 0, 18446744073709551615, 140737488355327, 18446744073709551615, 18446744073709551615, 18446744073709551615, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709486080, 562949953421311, 281474976710655, 18446744069414584446, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 576460752303423487, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073741823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 144115188075855871, 18446462600880324607, 9223372036854775807, 70368744112128, 281474976710655, 16212958624174047247, 65535, 0, 0, 0, 0, 0, 0, 35184372088831, 0, 0, 0, 18446744073709551615, 18014398505187016704, 1048575, 18446744073709551615, 67583, 4294443008, 34902944356761600, 18446744073709551615, 18446744073709551615, 18446744073709551615, 9223372036858970111, 2147483647, 0, 18446744073709551615, 2251799813685247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065665457643847680, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1125934266580991, 18446463629527547904, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1152921504606846975, 0, 0, 0, 0, 18446744073709551615, 2305570330330005503, 67043839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073707454463, 17005555242810474495, 18446744073709551599, 8935141660164089791, 18446744073709419615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446743249075830783, 17870283321271910397, 18437736874452713471, 18446603336221163519, 18446741874686295551, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8660801552383, 0, 0, 0, 18446462598732840960, 70368744177663, 0, 0, 4575692405780512767, 16384, 0, 0, 0, 0, 70368744112128, 17592186044415, 0, 0, 0, 0, 0, 0, 0, 17592185978880, 0, 0, 0, 351843720822784, 0, 0, 0, 13843853836919242751, 0, 0, 0, 9223213153129594880, 18446744073709551615, 18446744073709551615, 18446744073709551615, 31, 18446744073709551615, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790380184120328175, 6843210385291930244, 1152917029519358975, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 4294967295, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744070488326143, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446532967477018623, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446462607322775551, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1073741823, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 1073741823, 0, 0, 0, 0, 0, 0, 0, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709488127, 18446744073709551615, 18446744073709551615, 18446744073709551615, 288230376151711743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+#[inline]
+pub fn xid_start_p(c: char) -> bool {
+ let cp = c as usize;
+ let blki = unsafe { *XID_START_L1.get_unchecked(cp >> 10) } as usize;
+ let in_blk_offset_p = cp & 0x3FF;
+ let wordi = (blki * 16) + (in_blk_offset_p >> 6);
+ let biti = in_blk_offset_p & 0x3F;
+ unsafe { return (*XID_START_L2.get_unchecked(wordi) & (1 << biti)) != 0; }
+}
diff --git a/oryxc/src/unicode/line_terminator.rs b/oryxc/src/unicode/line_terminator.rs
index 5c6b052..300bb54 100644
--- a/oryxc/src/unicode/line_terminator.rs
+++ b/oryxc/src/unicode/line_terminator.rs
@@ -1,135 +1 @@
-/* Autogenerated – DO NOT EDIT */
-
-static LINE_TERMINATOR_L1: [u16; 544] = [
- 0, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-];
-static LINE_TERMINATOR_L2: [u64; 96] = [
- 7168,
- 0,
- 32,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 3298534883328,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
-];
-#[inline]
-pub fn line_terminator_p(c: char) -> bool {
- let cp = c as usize;
- let blki = unsafe { *LINE_TERMINATOR_L1.get_unchecked(cp >> 11) } as usize;
- let in_blk_offset_p = cp & 0x7FF;
- let wordi = (blki * 32) + (in_blk_offset_p >> 6);
- let biti = in_blk_offset_p & 0x3F;
- unsafe {
- return (*LINE_TERMINATOR_L2.get_unchecked(wordi) & (1 << biti)) != 0;
- }
-}
+include!(concat!(env!("OUT_DIR"), "/line_terminator.rs"));
diff --git a/oryxc/src/unicode/pattern_white_space.rs b/oryxc/src/unicode/pattern_white_space.rs
index b051e3a..f90db8d 100644
--- a/oryxc/src/unicode/pattern_white_space.rs
+++ b/oryxc/src/unicode/pattern_white_space.rs
@@ -1,137 +1 @@
-/* Autogenerated – DO NOT EDIT */
-
-static PATTERN_WHITE_SPACE_L1: [u16; 544] = [
- 0, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-];
-static PATTERN_WHITE_SPACE_L2: [u64; 96] = [
- 4294983168,
- 0,
- 32,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 3298534932480,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
-];
-#[inline]
-pub fn pattern_white_space_p(c: char) -> bool {
- let cp = c as usize;
- let blki =
- unsafe { *PATTERN_WHITE_SPACE_L1.get_unchecked(cp >> 11) } as usize;
- let in_blk_offset_p = cp & 0x7FF;
- let wordi = (blki * 32) + (in_blk_offset_p >> 6);
- let biti = in_blk_offset_p & 0x3F;
- unsafe {
- return (*PATTERN_WHITE_SPACE_L2.get_unchecked(wordi) & (1 << biti))
- != 0;
- }
-}
+include!(concat!(env!("OUT_DIR"), "/pattern_white_space.rs"));
diff --git a/oryxc/src/unicode/xid_continue.rs b/oryxc/src/unicode/xid_continue.rs
index 8fbbce4..571ae40 100644
--- a/oryxc/src/unicode/xid_continue.rs
+++ b/oryxc/src/unicode/xid_continue.rs
@@ -1,1007 +1 @@
-/* Autogenerated – DO NOT EDIT */
-
-static XID_CONTINUE_L1: [u16; 1088] = [
- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 12, 12, 12, 12, 12, 12, 13, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 14, 15, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 16, 9, 9, 9, 9, 9, 9,
- 9, 9, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 9, 29, 12, 30, 12,
- 12, 31, 32, 9, 9, 9, 9, 9, 9, 33, 9, 34, 35, 12, 12, 12, 12, 12, 12, 12,
- 36, 9, 9, 9, 9, 9, 9, 9, 37, 38, 9, 9, 39, 9, 9, 9, 40, 41, 42, 43, 44, 45,
- 46, 47, 48, 9, 9, 49, 9, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 50, 12, 12, 12, 12, 51, 12, 12, 12,
- 12, 52, 12, 12, 12, 12, 12, 12, 53, 54, 9, 9, 55, 9, 12, 12, 12, 12, 56,
- 12, 12, 12, 12, 12, 12, 12, 12, 57, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 58, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-];
-static XID_CONTINUE_L2: [u64; 944] = [
- 287948901175001088,
- 576460745995190270,
- 333270770471927808,
- 18410715276682199039,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 88094074470339,
- 18446744073709551615,
- 13321647697761927167,
- 18446744056529672128,
- 18428729675200069631,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709550843,
- 18446744073709551615,
- 18446462598732840959,
- 18446744069456527359,
- 13835058055282033151,
- 2119858418286774,
- 18446744069548736512,
- 18446678103011885055,
- 18446744073709551615,
- 11529212845433552895,
- 18446744073709486080,
- 18446744073709545471,
- 1125899906842623,
- 2612087783874887679,
- 70368744177663,
- 18446471390799331327,
- 18446744073701228287,
- 18446744056529682431,
- 18446744073709551615,
- 18446462392574410751,
- 17565725197581524975,
- 5765733215448889759,
- 15235112390417287150,
- 18014125208779143,
- 17576984196650090478,
- 18302910150157089727,
- 17576984196649951214,
- 844217444219295,
- 14123225865944680428,
- 281200107273671,
- 17582050746231021567,
- 281265452367327,
- 17577547146603651055,
- 4221916082617823,
- 18446744073709412351,
- 18158794964244397535,
- 3457638613854978030,
- 3658904103781503,
- 576460752303423486,
- 67076095,
- 4611685674830002134,
- 4093607775,
- 14024213633433600001,
- 18446216308128218879,
- 2305843009196916703,
- 64,
- 18446744073709551615,
- 18446744073709487103,
- 18446744070488326143,
- 17870283321406070975,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070446333439,
- 9168765891372858879,
- 18446744073701162813,
- 18446744073696837631,
- 1123704775901183,
- 18446744069414649855,
- 4557642822898941951,
- 18446744073709551614,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446638520593285119,
- 18446744069548802046,
- 144053615424700415,
- 9007197111451647,
- 3905461007941631,
- 18446744073709551615,
- 4394566287359,
- 18446744069481674752,
- 144115188075855871,
- 18446471394825863167,
- 18014398509481983,
- 1152657619668697087,
- 8796093022207936,
- 18446480190918885375,
- 134153215,
- 18446744069683019775,
- 11529215043920986111,
- 13834777130128311295,
- 17588964818943,
- 18446744073709551615,
- 4494803601399807,
- 18446744073709551615,
- 4503599627370495,
- 72057594037927935,
- 4611686018427380735,
- 16717361816799217663,
- 576460752302833664,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070475743231,
- 4611686017001275199,
- 6908521828386340863,
- 2295745090394464220,
- 9231253336202686464,
- 9223934986817634305,
- 536805376,
- 562821641207808,
- 17582049991377026180,
- 18446744069414601696,
- 511,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 4494940973301759,
- 18446498607738650623,
- 9223513873854758911,
- 9187201948305063935,
- 18446744071553646463,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 2251518330118602976,
- 18446744073709551614,
- 18446744068986765311,
- 18446744073709551615,
- 18446462598732840928,
- 18446744073709551615,
- 18446744069414617087,
- 18446462598732840960,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 8191,
- 4611686018427322368,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 17592185987071,
- 13830835930631503871,
- 18446744073709551615,
- 1125899906842623,
- 18446744060816261120,
- 18446744073709551615,
- 18446744073709550079,
- 18446181124293001215,
- 18691697672191,
- 4503599627370495,
- 18446744073709551615,
- 16789419406609285183,
- 18446532967477018623,
- 2305843004919775231,
- 18446744073709551615,
- 9223372032626884609,
- 36028797018963967,
- 18194542490348896255,
- 18446744073709551615,
- 35184368733388807,
- 18446602782178705022,
- 18446466996645134335,
- 18446744073709551615,
- 288010473826156543,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446462667452317695,
- 1152921504606845055,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446532967477018623,
- 18446744073709551615,
- 67108863,
- 6881498031078244479,
- 18446744073709551579,
- 1125899906842623,
- 18446744073709027328,
- 18446744073709551615,
- 18446744006063816703,
- 18446744073709551615,
- 18446744073709551615,
- 4611686018427387903,
- 18446744073709486080,
- 18446744073709355007,
- 287948901175001343,
- 7036870122864639,
- 12288634533233876992,
- 18446744073709551615,
- 2305843009213693951,
- 9799832780635308032,
- 18446743936404815870,
- 9223372036854775807,
- 486341884,
- 13258596753222922239,
- 1073692671,
- 18446744073709551615,
- 576460752303423487,
- 0,
- 9007199254740991,
- 0,
- 2305843009213693952,
- 0,
- 0,
- 18446744069951455231,
- 4295098367,
- 18446708893632430079,
- 576460752303359999,
- 18446744070488326143,
- 4128527,
- 18446744073709551615,
- 18446744073709551615,
- 18446466993558126591,
- 1152921504591118335,
- 18446463698244468735,
- 17870001915148894207,
- 2016486715970549759,
- 4503599627370495,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 36028797018963967,
- 1095220854783,
- 575897802350002111,
- 0,
- 10502394331027995967,
- 36028792728190975,
- 2147483647,
- 15762594400829440,
- 288230371860938751,
- 67108863,
- 13907115649320091647,
- 0,
- 9745789593611923567,
- 2305843004918726656,
- 536870911,
- 549755813631,
- 18014398509481983,
- 2251795522912255,
- 262143,
- 0,
- 18446744073709551615,
- 511,
- 2251799813685247,
- 2251799813685247,
- 287950000686628863,
- 18446671780820025343,
- 63,
- 0,
- 0,
- 0,
- 875211255709695,
- 18158513697557840124,
- 18446463149025525759,
- 18446462598732972031,
- 18446462598732841023,
- 36028792723996703,
- 18446744073709551615,
- 9241386160486350975,
- 576460752303423487,
- 287951100198191108,
- 18437736874454810623,
- 22517998136787184,
- 18446744073709551615,
- 402644511,
- 13907115649319829503,
- 3,
- 18446464796682337663,
- 287957697268023295,
- 18153444948953374703,
- 8760701963286943,
- 18428729675200023551,
- 25770850213,
- 18446744073709551615,
- 16173172735,
- 18446744073709551615,
- 67043519,
- 0,
- 0,
- 18392700878181105663,
- 1056964609,
- 18446744073709551615,
- 67043345,
- 144115188075855871,
- 68719412223,
- 287966492958392319,
- 127,
- 0,
- 0,
- 576460752303423487,
- 0,
- 18446744069414584320,
- 9223376434901286911,
- 17996384110963061375,
- 67043343,
- 18446740770879700992,
- 120208752639,
- 9223372036854775807,
- 18446744073709486208,
- 18446462599336820735,
- 144115188075855871,
- 0,
- 1095216660480,
- 0,
- 287948909764935679,
- 18410715276690587135,
- 18445618173869752321,
- 36027697507139583,
- 0,
- 13006395723845991295,
- 18446741595580465407,
- 18446466992517644287,
- 4394019979263,
- 0,
- 0,
- 0,
- 36028792723996672,
- 14411518807585456127,
- 134152199,
- 281474976710656,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 67108863,
- 0,
- 18446744073709551615,
- 140737488355327,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 15,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709486080,
- 562949953421311,
- 281474976710655,
- 18446744069418778623,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 576460752303423487,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 127,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 288230376151711743,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 144115188075855871,
- 18446466994631868415,
- 9223372036854775807,
- 8796093022143487,
- 36028797018963967,
- 16212958624241090575,
- 65535,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 287984085547089919,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18014398505187016704,
- 1048575,
- 18446744073709551615,
- 18446744073709520895,
- 4294934783,
- 35747438006370304,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 9223372036858970111,
- 2147483647,
- 0,
- 18446744073709551615,
- 2251799813685247,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 8065665457643847680,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1125934266580991,
- 18446463629527547904,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1152921504606846975,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 2305570330330005503,
- 1677656575,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 287948901175001088,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446532967477018623,
- 127,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 17872504197455282176,
- 65970697670631,
- 0,
- 0,
- 28,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073707454463,
- 17005555242810474495,
- 18446744073709551599,
- 8935141660164089791,
- 18446744073709419615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446743249075830783,
- 17870283321271910397,
- 18437736874452713471,
- 18446603336221163519,
- 18446741874686295551,
- 18446744073709539319,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 17906312118425092095,
- 9042383626829823,
- 281470547525648,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 8660801552383,
- 0,
- 0,
- 0,
- 18446471240106377087,
- 70368744177663,
- 32768,
- 0,
- 4611439727822766079,
- 17407,
- 0,
- 0,
- 0,
- 0,
- 140737488289792,
- 288230376151711743,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 288230376151646208,
- 0,
- 0,
- 0,
- 576460752303357952,
- 0,
- 0,
- 0,
- 13853072451644162047,
- 0,
- 0,
- 0,
- 9223213153129594880,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 8323103,
- 18446744073709551615,
- 67047423,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 790380184120328175,
- 6843210385291930244,
- 1152917029519358975,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 287948901175001088,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 4294967295,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070488326143,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446532967477018623,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446462607322775551,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1073741823,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1073741823,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709488127,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 288230376151711743,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 281474976710655,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
-];
-#[inline]
-pub fn xid_continue_p(c: char) -> bool {
- let cp = c as usize;
- let blki = unsafe { *XID_CONTINUE_L1.get_unchecked(cp >> 10) } as usize;
- let in_blk_offset_p = cp & 0x3FF;
- let wordi = (blki * 16) + (in_blk_offset_p >> 6);
- let biti = in_blk_offset_p & 0x3F;
- unsafe {
- return (*XID_CONTINUE_L2.get_unchecked(wordi) & (1 << biti)) != 0;
- }
-}
+include!(concat!(env!("OUT_DIR"), "/xid_continue.rs"));
diff --git a/oryxc/src/unicode/xid_start.rs b/oryxc/src/unicode/xid_start.rs
index 1c9d9ae..649faa5 100644
--- a/oryxc/src/unicode/xid_start.rs
+++ b/oryxc/src/unicode/xid_start.rs
@@ -1,927 +1 @@
-/* Autogenerated – DO NOT EDIT */
-
-static XID_START_L1: [u16; 1088] = [
- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 12, 12, 12, 12, 12, 12, 13, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 14, 15, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 16, 9, 9, 9, 9, 9, 9,
- 9, 9, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 9, 29, 12, 30, 12,
- 12, 31, 32, 9, 9, 9, 9, 9, 9, 33, 9, 34, 35, 12, 12, 12, 12, 12, 12, 12,
- 36, 9, 9, 9, 9, 9, 9, 9, 37, 38, 9, 9, 39, 9, 9, 9, 9, 9, 40, 9, 41, 42,
- 43, 44, 45, 9, 9, 9, 9, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
- 12, 12, 12, 12, 12, 12, 12, 12, 12, 46, 12, 12, 12, 12, 47, 12, 12, 12, 12,
- 48, 12, 12, 12, 12, 12, 12, 49, 50, 9, 9, 51, 9, 12, 12, 12, 12, 52, 12,
- 12, 12, 12, 12, 12, 12, 12, 53, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-];
-static XID_START_L2: [u64; 864] = [
- 68719476736,
- 576460745995190270,
- 297241973452963840,
- 18410715276682199039,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 88094074470339,
- 0,
- 13321366222785216512,
- 18446744056529672000,
- 18428729675200069631,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709550595,
- 18446744073709551615,
- 18446462598732840959,
- 18446744069456527359,
- 511,
- 2119858418286592,
- 18446744069414584320,
- 18446392229988665343,
- 18446744073709551615,
- 11241196188469297151,
- 281474976514048,
- 18446744073709543424,
- 563224831328255,
- 301749971126844416,
- 1168302407679,
- 18446471390564450303,
- 18446744069414649599,
- 1023,
- 2594073385365405680,
- 18446181140919287808,
- 2577745637692514273,
- 1153765945374687232,
- 247132830528276448,
- 7881300924956672,
- 2589004636761079776,
- 144115200960823296,
- 2589004636760940512,
- 562965791113216,
- 288167810662516712,
- 65536,
- 2594071186342010848,
- 13807648768,
- 2589567586714640353,
- 1688864624214016,
- 2882303761516978160,
- 18158513712597581824,
- 3457638613854978016,
- 127,
- 1688849860263934,
- 127,
- 2307531515476572118,
- 4026531935,
- 1,
- 35184372088575,
- 7936,
- 0,
- 9223380832947798015,
- 18438229877581611008,
- 18446744069414600707,
- 17870283321406070975,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070446333439,
- 9168765891372858879,
- 18446744073701162813,
- 18446744073696837631,
- 134217727,
- 18446744069414649855,
- 4557642822898941951,
- 18446744073709551614,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446638520593285119,
- 18446744069548802046,
- 144053615424700415,
- 1125897759621119,
- 527761286627327,
- 4503599627370495,
- 276824064,
- 18446744069414584320,
- 144115188075855871,
- 18446469195802607615,
- 18014398509481983,
- 2147483647,
- 8796093022142464,
- 18446480190918885375,
- 1023,
- 18446744069422972927,
- 2097151,
- 549755813888,
- 0,
- 4503599627370464,
- 8160,
- 18158724812380307448,
- 274877906943,
- 68719476735,
- 4611686018360336384,
- 16717361816799217663,
- 319718190147960832,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070475743231,
- 4611686017001275199,
- 6908521828386340863,
- 2295745090394464220,
- 0,
- 9223934986808197120,
- 536805376,
- 0,
- 17582049991377026180,
- 18446744069414601696,
- 511,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 3509778554814463,
- 18446498607738650623,
- 141836999983103,
- 9187201948305063935,
- 2139062143,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 2251241253188403424,
- 18446744073709551614,
- 18446744068886102015,
- 17870283321406128127,
- 18446462598732840928,
- 18446744073709551615,
- 18446744069414617087,
- 18446462598732840960,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 8191,
- 4611686018427322368,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 13198434443263,
- 9223512774343131135,
- 18446744070488326143,
- 281474976710655,
- 18446744060816261120,
- 18446744073709551615,
- 18446744073709550079,
- 18446181124293001215,
- 34359736251,
- 4503599627370495,
- 4503599627370492,
- 7564921474075590656,
- 18446462873610746880,
- 2305843004918726783,
- 2251799813685232,
- 8935422993945886720,
- 2199023255551,
- 14159317224157876215,
- 4495436853045886975,
- 7890092085477381,
- 18446602782178705022,
- 18446466996645134335,
- 18446744073709551615,
- 34359738367,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446462667452317695,
- 1152921504606845055,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446532967477018623,
- 18446744073709551615,
- 67108863,
- 6881498030004502655,
- 18446744073709551579,
- 1125899906842623,
- 18446744073709027328,
- 18446744073709551615,
- 18446744006063816703,
- 18446744073709551615,
- 18446744073709551615,
- 4611686018427387903,
- 18446744073709486080,
- 18446744073709355007,
- 287948901175001343,
- 0,
- 12288634533233819648,
- 18446744073709551615,
- 2305843009213693951,
- 576460743713488896,
- 18446743798965862398,
- 9223372033633550335,
- 486341884,
- 13258596753222922239,
- 1073692671,
- 18446744073709551615,
- 576460752303423487,
- 0,
- 9007199254740991,
- 0,
- 0,
- 0,
- 0,
- 18446744069951455231,
- 131071,
- 18446708893632430079,
- 18014398509418495,
- 18446744070488326143,
- 4128527,
- 18446744073709551615,
- 18446744073709551615,
- 18446462599806582783,
- 1152921504591118335,
- 18446463698244468735,
- 17870001915148894207,
- 2016486715970549759,
- 4503599627370495,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 36028797018963967,
- 1095220854783,
- 575897802350002111,
- 0,
- 10502394331027995967,
- 36028792728190975,
- 2147483647,
- 15762594400829440,
- 288230371860938751,
- 67108863,
- 13907115649320091647,
- 0,
- 18014398491590657,
- 2305843004918726656,
- 536870911,
- 137438953215,
- 18014398509481983,
- 2251795522912255,
- 262143,
- 0,
- 18446744073709551615,
- 511,
- 2251799813685247,
- 2251799813685247,
- 68719476735,
- 18446603611099102208,
- 63,
- 0,
- 0,
- 0,
- 848822976643071,
- 252,
- 18446463149025525759,
- 18446462598732841023,
- 18446462598732840963,
- 36028792723996703,
- 72057594037927928,
- 10696049115004928,
- 281474976710648,
- 2199023190016,
- 549755813880,
- 20266198323101840,
- 2251799813685240,
- 335544350,
- 9223389629040558079,
- 1,
- 18446464796682337663,
- 2147483647,
- 2589004636760940512,
- 16643063808,
- 54043195528399871,
- 655360,
- 9007199254740991,
- 15032387456,
- 281474976710655,
- 176,
- 0,
- 0,
- 140737488355327,
- 251658240,
- 281474976710655,
- 16,
- 72066390130950143,
- 0,
- 134217727,
- 127,
- 0,
- 0,
- 17592186044415,
- 0,
- 18446744069414584320,
- 9223372041149743103,
- 9223653511822045823,
- 2,
- 18446740770879700992,
- 42949804031,
- 290482175965394945,
- 18446744073441181696,
- 18446462599269712895,
- 144115188075855871,
- 0,
- 0,
- 0,
- 8589934591,
- 140737488354815,
- 18445618173802708993,
- 65535,
- 0,
- 562949953420159,
- 18446741595513421888,
- 18446462598749619199,
- 268435455,
- 0,
- 0,
- 0,
- 2251795518717952,
- 4503599627239412,
- 0,
- 281474976710656,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 67108863,
- 0,
- 18446744073709551615,
- 140737488355327,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 15,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709486080,
- 562949953421311,
- 281474976710655,
- 18446744069414584446,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 576460752303423487,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 127,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 1073741823,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 144115188075855871,
- 18446462600880324607,
- 9223372036854775807,
- 70368744112128,
- 281474976710655,
- 16212958624174047247,
- 65535,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 35184372088831,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18014398505187016704,
- 1048575,
- 18446744073709551615,
- 67583,
- 4294443008,
- 34902944356761600,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 9223372036858970111,
- 2147483647,
- 0,
- 18446744073709551615,
- 2251799813685247,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 8065665457643847680,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1125934266580991,
- 18446463629527547904,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1152921504606846975,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 2305570330330005503,
- 67043839,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073707454463,
- 17005555242810474495,
- 18446744073709551599,
- 8935141660164089791,
- 18446744073709419615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446743249075830783,
- 17870283321271910397,
- 18437736874452713471,
- 18446603336221163519,
- 18446741874686295551,
- 4087,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 8660801552383,
- 0,
- 0,
- 0,
- 18446462598732840960,
- 70368744177663,
- 0,
- 0,
- 4575692405780512767,
- 16384,
- 0,
- 0,
- 0,
- 0,
- 70368744112128,
- 17592186044415,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 17592185978880,
- 0,
- 0,
- 0,
- 351843720822784,
- 0,
- 0,
- 0,
- 13843853836919242751,
- 0,
- 0,
- 0,
- 9223213153129594880,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 31,
- 18446744073709551615,
- 2063,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 790380184120328175,
- 6843210385291930244,
- 1152917029519358975,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 4294967295,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744070488326143,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446532967477018623,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446462607322775551,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1073741823,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 1073741823,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709488127,
- 18446744073709551615,
- 18446744073709551615,
- 18446744073709551615,
- 288230376151711743,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
-];
-#[inline]
-pub fn xid_start_p(c: char) -> bool {
- let cp = c as usize;
- let blki = unsafe { *XID_START_L1.get_unchecked(cp >> 10) } as usize;
- let in_blk_offset_p = cp & 0x3FF;
- let wordi = (blki * 16) + (in_blk_offset_p >> 6);
- let biti = in_blk_offset_p & 0x3F;
- unsafe {
- return (*XID_START_L2.get_unchecked(wordi) & (1 << biti)) != 0;
- }
-}
+include!(concat!(env!("OUT_DIR"), "/xid_start.rs"));
diff --git a/unigen/Cargo.toml b/unigen/Cargo.toml
deleted file mode 100644
index 587afb7..0000000
--- a/unigen/Cargo.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-[package]
-name = "unigen"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-lexopt = "0.3.2"
diff --git a/unigen/fetch b/unigen/fetch
deleted file mode 100755
index 46c02f9..0000000
--- a/unigen/fetch
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd "${0%/*}"
-trap 'rm -f UCD.zip' EXIT
-
-set -x
-mkdir -p data
-curl -LO https://www.unicode.org/Public/zipped/latest/UCD.zip
-unzip -od data UCD.zip
-
-# XID_Start and XID_Continue additions
-cat <<-EOF >>data/DerivedCoreProperties.txt
-0024 ; XID_Start # Pc DOLLAR SIGN
-005F ; XID_Start # Pc LOW LINE
-2032..2034 ; XID_Continue # Po [3] PRIME..TRIPLE PRIME
-2057 ; XID_Continue # Po QUADRUPLE PRIME
-EOF
diff --git a/unigen/src/main.rs b/unigen/src/main.rs
deleted file mode 100644
index bedabc6..0000000
--- a/unigen/src/main.rs
+++ /dev/null
@@ -1,279 +0,0 @@
-use std::collections::HashMap;
-use std::ffi::{
- OsStr,
- OsString,
-};
-use std::fs::File;
-use std::io::{
- self,
- BufRead,
- BufReader,
-};
-use std::path::Path;
-use std::sync::OnceLock;
-use std::vec::Vec;
-use std::{
- env,
- process,
-};
-
-const MIN_SHIFT: usize = 1;
-const MAX_SHIFT: usize = 22;
-
-#[derive(Default)]
-struct Flags {
- codepoints: Option<Vec<char>>,
- help: bool,
-}
-
-impl Flags {
- fn parse() -> Result<(Flags, Vec<String>), lexopt::Error> {
- use lexopt::prelude::*;
-
- let mut rest = Vec::with_capacity(env::args().len() - 1);
- let mut flags = Flags::default();
- let mut parser = lexopt::Parser::from_env();
- parser.set_short_equals(false);
-
- while let Some(arg) = parser.next()? {
- match arg {
- Short('c') | Long("codepoints") => {
- fn hex_to_char(s: &str) -> char {
- return u32::from_str_radix(s, 16).map_or_else(
- |e| {
- eprintln!("{}: {s}: {e}", progname().display());
- process::exit(1);
- },
- |n| {
- char::from_u32(n).unwrap_or_else(|| {
- eprintln!(
- "{}: {s}: invalid codepoint",
- progname().display()
- );
- process::exit(1);
- })
- },
- );
- }
-
- flags.codepoints = Some(
- parser
- .value()?
- .to_str()
- .unwrap_or_else(|| {
- eprintln!(
- "{}: unable to parse argument to -c/--codepoints",
- progname().display()
- );
- process::exit(1);
- })
- .split(',')
- .map(hex_to_char)
- .collect(),
- );
- },
- Short('h') | Long("help") => flags.help = true,
- Value(v) => rest.push(v.into_string()?),
- _ => return Err(arg.unexpected()),
- }
- }
-
- return Ok((flags, rest));
- }
-}
-
-fn progname() -> &'static OsString {
- static ARGV0: OnceLock<OsString> = OnceLock::new();
- return ARGV0.get_or_init(|| {
- let default = OsStr::new("oryxc");
- let s = env::args_os().next().unwrap_or(default.into());
- return Path::new(&s).file_name().unwrap_or(default).to_os_string();
- });
-}
-
-fn usage() {
- eprintln!(
- concat!(
- "Usage: {0} data-file property-name\n",
- " {0} -c codepoints name\n",
- " {0} -h",
- ),
- progname().display()
- );
-}
-
-fn main() -> io::Result<()> {
- let (flags, rest) = match Flags::parse() {
- Ok(v) => v,
- Err(e) => {
- eprintln!("{}: {e}", progname().display());
- usage();
- process::exit(1);
- },
- };
-
- if flags.help {
- usage();
- process::exit(0);
- }
-
- if (flags.codepoints.is_none() && rest.len() != 2)
- || (flags.codepoints.is_some() && rest.len() != 1)
- {
- usage();
- process::exit(1);
- }
-
- let mut bitmap = vec![false; 0x110000];
- let name = match flags.codepoints {
- Some(vec) => {
- vec.iter().for_each(|c| bitmap[*c as usize] = true);
- &rest[0]
- },
- None => {
- parse_file(&rest[0], &rest[1], &mut bitmap)?;
- &rest[1]
- },
- };
- let (shift, lvl1, lvl2) = optimize_tables(&bitmap);
- write_tables(name, shift, &lvl1, &lvl2);
- return Ok(());
-}
-
-fn optimize_tables(bitmap: &[bool]) -> (usize, Vec<u16>, Vec<u64>) {
- let mut minsz = usize::MAX;
- let mut config = (0, Vec::new(), Vec::new());
-
- for i in MIN_SHIFT..=MAX_SHIFT {
- let (l1, l2) = build_tables(bitmap, i);
- let sz = l1.len() * 2 + l2.len() * 8;
- if sz < minsz {
- minsz = sz;
- config = (i, l1, l2);
- }
- }
-
- return config;
-}
-
-fn parse_file<P: AsRef<Path>>(
- path: P,
- prop: &str,
- bitmap: &mut [bool],
-) -> io::Result<()> {
- let file = File::open(path)?;
- let reader = BufReader::new(file);
-
- for line in reader.lines() {
- let line = line?;
- let line = line.split('#').next().unwrap_or("").trim();
- if line.is_empty() {
- continue;
- }
-
- let parts: Vec<&str> = line.split(';').map(|s| s.trim()).collect();
- if parts.len() < 2 || parts[1] != prop {
- continue;
- }
-
- let (beg, end) = if parts[0].contains("..") {
- let mut range = parts[0].split("..");
- (
- u32::from_str_radix(range.next().unwrap(), 16).unwrap(),
- u32::from_str_radix(range.next().unwrap(), 16).unwrap(),
- )
- } else {
- let val = u32::from_str_radix(parts[0], 16).unwrap();
- (val, val)
- };
-
- for cp in beg..=end {
- if (cp as usize) < bitmap.len() {
- bitmap[cp as usize] = true;
- }
- }
- }
- return Ok(());
-}
-
-fn build_tables(bitmap: &[bool], shift: usize) -> (Vec<u16>, Vec<u64>) {
- let blksz = 1 << shift;
- let u64s_per_block = (blksz + 63) / 64;
-
- let mut lvl2: Vec<u64> = Vec::new();
- let mut lvl1: Vec<u16> = Vec::new();
- let mut blkmap: HashMap<Vec<u64>, u16> = HashMap::new();
-
- for chunk in bitmap.chunks(blksz) {
- let mut blkdata = vec![0u64; u64s_per_block];
-
- for (i, &bit) in chunk.iter().enumerate() {
- if bit {
- let word_idx = i / 64;
- let bit_idx = i % 64;
- blkdata[word_idx] |= 1 << bit_idx;
- }
- }
-
- if let Some(&i) = blkmap.get(&blkdata) {
- lvl1.push(i);
- } else {
- let i = (lvl2.len() / u64s_per_block) as u16;
- lvl2.extend_from_slice(&blkdata);
- blkmap.insert(blkdata, i);
- lvl1.push(i);
- }
- }
-
- return (lvl1, lvl2);
-}
-
-fn write_tables(prop_name: &str, shift: usize, level1: &[u16], level2: &[u64]) {
- let upper_name = prop_name.to_uppercase();
- let lower_name = prop_name.to_lowercase();
- let block_size = 1 << shift;
- let mask = block_size - 1;
- let u64s_per_block = (block_size + 63) / 64;
-
- println!("/* Autogenerated – DO NOT EDIT */\n");
- print!(
- "static {upper_name}_L1: [u16; {}] = {level1:?};",
- level1.len()
- );
- print!(
- "static {upper_name}_L2: [u64; {}] = {level2:?};",
- level2.len()
- );
-
- let pred_name = if lower_name.contains('_') {
- format!("{lower_name}_p")
- } else {
- format!("{lower_name}p")
- };
-
- print!(
- "#[inline]
- pub fn {pred_name}(c: char) -> bool {{
- let cp = c as usize;
- let blki = unsafe {{ *{upper_name}_L1.get_unchecked(cp >> {shift}) }} as usize;
- let in_blk_offset_p = cp & 0x{mask:X};"
- );
-
- if u64s_per_block == 1 {
- print!(
- " unsafe {{
- return ({upper_name}_L2.get_unchecked(blki) & (1 << in_blk_offset_p)) != 0;
- }}"
- );
- } else {
- print!(
- "let wordi = (blki * {u64s_per_block}) + (in_blk_offset_p >> 6);
- let biti = in_blk_offset_p & 0x3F;
- unsafe {{
- return (*{upper_name}_L2.get_unchecked(wordi) & (1 << biti)) != 0;
- }}"
- );
- }
-
- print!("}}");
-}