hare

The Hare programming language
git clone https://git.torresjrjr.com/hare.git
Log | Files | Refs | README | LICENSE

commit 04eb6b95b83fa1107ebc873d704f5a75adc73b21
parent 04e8af8c7bf18361766017738c48f11cf239e5cf
Author: Bor Grošelj Simić <bor.groseljsimic@telemach.net>
Date:   Mon,  8 Mar 2021 23:59:30 +0100

tokenizer: return zero tokens on empty slice

Diffstat:
Mbytes/tokenize.ha | 7+++++++
Mpath/iter.ha | 8++++----
Mstrings/tokenize.ha | 4++++
3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/bytes/tokenize.ha b/bytes/tokenize.ha @@ -7,6 +7,9 @@ export type tokenizer = struct { s: []u8, d: []u8, p: size }; // Caller should ensure delim is not an empty slice export fn tokenize(s: []u8, delim: []u8) tokenizer = { assert(len(delim) > 0); + if (len(s) == 0) { + delim = []; + }; return tokenizer { s = s, d = delim, @@ -153,4 +156,8 @@ export fn remaining_tokens(s: *tokenizer) []u8 = { assert(equal(remaining_tokens(&t), [2, 3, 4])); assert(equal(peek_token(&t) as []u8, [2, 3, 4])); assert(equal(remaining_tokens(&t), [2, 3, 4])); + + t = tokenize([]: []u8, [42]); + assert(peek_token(&t) is void); + assert(next_token(&t) is void); }; diff --git a/path/iter.ha b/path/iter.ha @@ -69,8 +69,8 @@ export fn next(iter: *iterator) (path | void) = { let i = iter("foo"); assert(equal(next(&i) as path, "foo")); assert(next(&i) is void); - // Hm? - //let i = iter("/"); - //assert(equal(next(&i) as path, "/")); - //assert(next(&i) is void); + + let i = iter("/"); + assert(equal(next(&i) as path, "/")); + assert(next(&i) is void); }; diff --git a/strings/tokenize.ha b/strings/tokenize.ha @@ -74,6 +74,10 @@ export fn remaining_tokens(s: *tokenizer) str = { assert(peek_token(&tok) is void); assert(next_token(&tok) is void); + + tok = tokenize("", "foo"); + assert(peek_token(&tok) is void); + assert(next_token(&tok) is void); }; // Splits a string into tokens delimited by 'delim', returning a slice of up to