summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOri Bernstein <ori@eigenstate.org>2018-12-26 21:43:45 -0800
committerOri Bernstein <ori@eigenstate.org>2018-12-26 21:43:45 -0800
commit346140113cb979d60f27fb3795f08e5976c0c712 (patch)
treef86b1d17a69a7469f8c2f9bb1d6c3342b28d8103
parent3f5de2514eb4a90a8f7925bbeaf6dfdfb07debf8 (diff)
downloadmc-346140113cb979d60f27fb3795f08e5976c0c712.tar.gz
Implement `std.bygrapheme`.
-rw-r--r--lib/std/striter.myr80
-rw-r--r--lib/std/test/striter.myr18
2 files changed, 90 insertions, 8 deletions
diff --git a/lib/std/striter.myr b/lib/std/striter.myr
index f786e2c..dce382c 100644
--- a/lib/std/striter.myr
+++ b/lib/std/striter.myr
@@ -4,12 +4,18 @@ use "utf"
use "strfind"
use "option"
use "chartype"
+use "slpush"
+use "alloc"
pkg std =
type chariter = struct
rest : byte[:]
;;
+ type graphemeiter = struct
+ rest : byte[:]
+ ;;
+
type charoffiter = struct
str : byte[:]
idx : size
@@ -25,17 +31,24 @@ pkg std =
idx : size
;;
- impl iterable chariter -> char
+ impl iterable chariter -> char
+ impl iterable graphemeiter -> char[:]
impl iterable charoffiter -> (char, size)
- impl iterable splititer -> byte[:]
- impl iterable tokiter -> byte[:]
-
- const bychar : (str : byte[:] -> chariter)
- const bycharoff : (str : byte[:] -> charoffiter)
- const bysplit : (str : byte[:], split : byte[:] -> splititer)
- const bytok : (str : byte[:] -> tokiter)
+ impl iterable splititer -> byte[:]
+ impl iterable tokiter -> byte[:]
+
+ const bychar : (str : byte[:] -> chariter)
+ const bygrapheme : (str : byte[:] -> graphemeiter)
+ const bycharoff : (str : byte[:] -> charoffiter)
+ const bysplit : (str : byte[:], split : byte[:] -> splititer)
+ const bytok : (str : byte[:] -> tokiter)
;;
+/*
+ * Iterate through a string char by char,
+ * decoding the utf8 bytes into a single
+ * codepoint.
+ */
impl iterable chariter -> char =
__iternext__ = {ci, c
if ci.rest.len == 0
@@ -53,7 +66,50 @@ const bychar = {str
-> [.rest = str]
}
+/*
+ * Iterate through a string grapheme by grapheme,
+ * returning a slice of characters composing the
+ * grapheme.
+ */
+impl iterable graphemeiter -> char[:] =
+ __iternext__ = {ci, g : char[:]#
+ var gb, gc : char[:]
+ if ci.rest.len == 0
+ -> false
+ ;;
+ (gb, ci.rest) = graphemestep(ci.rest)
+
+ /*
+ * Graphemestep returns bytes, but we
+ * want to a slice of chars.
+ */
+ gc = [][:]
+ for c : std.bychar(gb)
+ std.slpush(&gc, c)
+ ;;
+ g# = gc
+ -> true
+ }
+
+ __iterfin__ = {ci, g
+ std.slfree(g#)
+ }
+;;
+
+const bygrapheme = {str
+ -> [.rest = str]
+}
+
+/*
+ * Iterates through a string character by
+ * character, similar to chariter, but returns
+ * the offset into the string of the codepoint.
+ * For example,
+ * "ὐbὐc
+ * would return the sequence:
+ * (ὐ, 0), (b, 3), (ὐ, 4), (c, 7)
+ */
impl iterable charoffiter -> (char, size) =
__iternext__ = {ci, cv
var c
@@ -75,6 +131,10 @@ const bycharoff = {s
-> [.str=s, .idx=0]
}
+/*
+ * Iterates through the splits of a string by a
+ * delimiter, skippin gthe delimiter.
+ */
impl iterable splititer -> byte[:] =
__iternext__ = {si, sp
match std.strfind(si.rest, si.split)
@@ -100,6 +160,10 @@ const bysplit = {str, split
-> [.rest = str, .split = split]
}
+/*
+ * Tokenizes a string by spaces, iterating over
+ * the results.
+ */
impl iterable tokiter -> byte[:] =
__iternext__ = {it, sp
var s, lo, hi, c
diff --git a/lib/std/test/striter.myr b/lib/std/test/striter.myr
index b54e580..0f93fb1 100644
--- a/lib/std/test/striter.myr
+++ b/lib/std/test/striter.myr
@@ -3,6 +3,18 @@ use std
const main = {
var chars = ['a', 'b', 'c']
var splits = ["foo", "+bar"]
+ var graphemes = [
+ [0x300][:],
+ [0x61][:],
+ [0x53f2][:],
+ [0x63][:],
+ [0x9][:],
+ [0x42f][:],
+ [0x78, 0x300, 0x300, 0x300, 0x300, 0x300][:],
+ [0xa][:],
+ [0x7a, 0x309][:]
+ ]
+
var i
i = 0
@@ -16,4 +28,10 @@ const main = {
std.assert(std.eq(splits[i++], sp), "wrong split {}", sp)
;;
std.assert(i == splits.len, "wrong split count")
+
+ i = 0
+ for g : std.bygrapheme("̀a史c\tЯx̀̀̀̀̀\nz̉")
+ std.assert(std.eq(g, graphemes[i++]), "mismatched grapheme cluster\n")
+ ;;
+ std.assert(i == graphemes.len, "wrong grapheme set length")
}