Add 'strtok()' implementation to split on spaces.

2 files changed, 49 insertions, 0 deletions

diff --git a/lib/std/strsplit.myr b/lib/std/strsplit.myr
index e64d907..f7c30d8 100644
--- a/lib/std/strsplit.myr
+++ b/lib/std/strsplit.myr
@@ -1,15 +1,20 @@
 use "alloc.use"
+use "chartype.use"
 use "die.use"
 use "extremum.use"
 use "option.use"
 use "slpush.use"
 use "strfind.use"
 use "types.use"
+use "utf.use"
 
 pkg std =
 	const strsplit	: (s : byte[:], delim : byte[:] -> byte[:][:])
+	const strtok	: (s : byte[:] -> byte[:][:])
 ;;
 
+extern const put	: (fmt : byte[:], args : ... -> size)
+
 const strsplit = {s, delim
 	var last
 	var sp
@@ -33,3 +38,24 @@ const strsplit = {s, delim
 	-> sp
 }
 
+const strtok = {s
+	var i, j
+	var toks
+
+	i = 0
+	toks = [][:]
+	while i != s.len
+		while isspace(std.decode(s[i:])) && i < s.len
+			i++
+		;;
+		j = i
+		while !std.isspace(std.decode(s[j:])) && j < s.len
+			j++
+		;;
+		if i != j
+			toks = slpush(toks, s[i:j])
+		;;
+		i = j
+	;;
+	-> toks
+}
diff --git a/lib/std/test/strsplit.myr b/lib/std/test/strsplit.myr
new file mode 100644
index 0000000..8c4a97c
--- /dev/null
+++ b/lib/std/test/strsplit.myr
@@ -0,0 +1,23 @@
+use std
+
+const main = {
+	check(std.strsplit("", ","), [][:])
+	check(std.strsplit("a,b,c ,,d,", ","), \
+		["a", "b", "c ", "", "d", ""][:])
+	check(std.strtok(""), [][:])
+	check(std.strtok(" "), [][:])
+	check(std.strtok("\t"), [][:])
+	check(std.strtok("a b  c\td"), ["a", "b", "c", "d"][:])
+}
+
+const check = {a, b
+	if a.len != b.len
+		std.fatal("a = {}, b = {}\n", a, b)
+		std.fatal("length mismatch: {} != {}: {}\n", a.len, b.len)
+	;;
+	for var i = 0; i < a.len; i++
+		if !std.sleq(a[i], b[i])
+			std.fatal("element {} mismatched: {} != {}\n", i, a[i], b[i])
+		;;
+	;;
+}