hare

commit 4397f94bbe1dc6b5d6377f9a9ddc5b0d5b76ed71
parent f07516e53209b36f9c1c97245b5b3efa77bbdf34
Author: Ember Sawady <ecs@d2evs.net>
Date:   Sun, 12 Feb 2023 23:50:14 +0000

strings::tokenizer: use strings::fromutf8_unsafe

The source is a str, so it's already guaranteed to be valid utf8

Signed-off-by: Ember Sawady <ecs@d2evs.net>

Diffstat:
M
strings/tokenize.ha
 | 
6
+++---

1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/strings/tokenize.ha b/strings/tokenize.ha
@@ -41,7 +41,7 @@ export fn rtokenize(s: str, delim: str) tokenizer =
 export fn next_token(s: *tokenizer) (str | void) = {
 	return match (bytes::next_token(s)) {
 	case let b: []u8 =>
-		yield fromutf8(b)!;
+		yield fromutf8_unsafe(b);
 	case void => void;
 	};
 };
@@ -50,7 +50,7 @@ export fn next_token(s: *tokenizer) (str | void) = {
 export fn peek_token(s: *tokenizer) (str | void) = {
 	return match (bytes::peek_token(s)) {
 	case let b: []u8 =>
-		yield fromutf8(b)!;
+		yield fromutf8_unsafe(b);
 	case void => void;
 	};
 };
@@ -58,7 +58,7 @@ export fn peek_token(s: *tokenizer) (str | void) = {
 // Returns the remainder of the string associated with a tokenizer, without doing
 // any further tokenization.
 export fn remaining_tokens(s: *tokenizer) str = {
-	return fromutf8(bytes::remaining_tokens(s))!;
+	return fromutf8_unsafe(bytes::remaining_tokens(s));
 };
 
 @test fn tokenize() void = {