hare

commit 04eb6b95b83fa1107ebc873d704f5a75adc73b21
parent 04e8af8c7bf18361766017738c48f11cf239e5cf
Author: Bor Grošelj Simić <bor.groseljsimic@telemach.net>
Date:   Mon,  8 Mar 2021 23:59:30 +0100

tokenizer: return zero tokens on empty slice

Diffstat:
M
bytes/tokenize.ha
 | 
7
+++++++
M
path/iter.ha
 | 
8
++++----
M
strings/tokenize.ha
 | 
4
++++

3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/bytes/tokenize.ha b/bytes/tokenize.ha
@@ -7,6 +7,9 @@ export type tokenizer = struct { s: []u8, d: []u8, p: size };
 // Caller should ensure delim is not an empty slice
 export fn tokenize(s: []u8, delim: []u8) tokenizer = {
 	assert(len(delim) > 0);
+	if (len(s) == 0) {
+		delim = [];
+	};
 	return tokenizer {
 		s = s,
 		d = delim,
@@ -153,4 +156,8 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
 	assert(equal(remaining_tokens(&t), [2, 3, 4]));
 	assert(equal(peek_token(&t) as []u8, [2, 3, 4]));
 	assert(equal(remaining_tokens(&t), [2, 3, 4]));
+
+	t = tokenize([]: []u8, [42]);
+	assert(peek_token(&t) is void);
+	assert(next_token(&t) is void);
 };
diff --git a/path/iter.ha b/path/iter.ha
@@ -69,8 +69,8 @@ export fn next(iter: *iterator) (path | void) = {
 	let i = iter("foo");
 	assert(equal(next(&i) as path, "foo"));
 	assert(next(&i) is void);
-	// Hm?
-	//let i = iter("/");
-	//assert(equal(next(&i) as path, "/"));
-	//assert(next(&i) is void);
+
+	let i = iter("/");
+	assert(equal(next(&i) as path, "/"));
+	assert(next(&i) is void);
 };
diff --git a/strings/tokenize.ha b/strings/tokenize.ha
@@ -74,6 +74,10 @@ export fn remaining_tokens(s: *tokenizer) str = {
 
 	assert(peek_token(&tok) is void);
 	assert(next_token(&tok) is void);
+
+	tok = tokenize("", "foo");
+	assert(peek_token(&tok) is void);
+	assert(next_token(&tok) is void);
 };
 
 // Splits a string into tokens delimited by 'delim', returning a slice of up to