Add a test for word breaking

1 files changed, 11 insertions, 0 deletions

diff --git a/test/gen-test-data b/test/gen-test-data
new file mode 100755
index 0000000..12ee11e
--- /dev/null
+++ b/test/gen-test-data
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+download()
+{
+	curl -sS "https://www.unicode.org/Public/15.1.0/ucd/$1"
+}
+
+download 'auxiliary/WordBreakTest.txt' \
+| sed -En 's/\s+//g; s/÷?#.*//g; /./p' >wnext.in