hare

commit 7a434284d30e97958300d47593c91fb8b7fc7340
parent 3325561cd0358974dea87cc0b00065603688392b
Author: Drew DeVault <sir@cmpwn.com>
Date:   Tue, 20 Apr 2021 08:52:51 -0400

hare::lex: merge repeated strings into one token

Diffstat:
M
hare/lex/+test.ha
 | 
24
++++++++----------------
M
hare/lex/lex.ha
 | 
14
++++++++++++++

2 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/hare/lex/+test.ha b/hare/lex/+test.ha
@@ -208,18 +208,7 @@ fn loc(line: uint, col: uint) location = location {
 	const in = "\"a\" \"b\" \"\\a\" \"\\b\" \"\\f\" \"\\n\" \"\\r\" "
 		"\"\\t\" \"\\v\" \"\\0\" \"\\\\\" \"\\\'\"";
 	const expected: [_]token = [
-		(ltok::LIT_STR, "a", loc(1, 1)),
-		(ltok::LIT_STR, "b", loc(1, 5)),
-		(ltok::LIT_STR, "\a", loc(1, 9)),
-		(ltok::LIT_STR, "\b", loc(1, 14)),
-		(ltok::LIT_STR, "\f", loc(1, 19)),
-		(ltok::LIT_STR, "\n", loc(1, 24)),
-		(ltok::LIT_STR, "\r", loc(1, 29)),
-		(ltok::LIT_STR, "\t", loc(1, 34)),
-		(ltok::LIT_STR, "\v", loc(1, 39)),
-		(ltok::LIT_STR, "\0", loc(1, 44)),
-		(ltok::LIT_STR, "\\", loc(1, 49)),
-		(ltok::LIT_STR, "\'", loc(1, 54)),
+		(ltok::LIT_STR, "ab\a\b\f\n\r\t\v\0\\\'", loc(1, 1)),
 	];
 	// TODO: test \x and \u and \U
 	lextest(in, expected);
@@ -228,12 +217,15 @@ fn loc(line: uint, col: uint) location = location {
 		(ltok::LIT_STR, "ab\a\b\f\n\r\t\v\0\\\'", loc(1, 1)),
 	];
 	lextest(in, expected);
-	const in = "\"hello world\" \"こんにちは\" \"return\" \"foo\"";
+	const in = "\"hello world\", \"こんにちは\", \"return\", \"foo\"";
 	const expected: [_]token = [
 		(ltok::LIT_STR, "hello world", loc(1, 1)),
-		(ltok::LIT_STR, "こんにちは", loc(1, 15)),
-		(ltok::LIT_STR, "return", loc(1, 23)),
-		(ltok::LIT_STR, "foo", loc(1, 32)),
+		(ltok::COMMA, void, loc(1, 15)),
+		(ltok::LIT_STR, "こんにちは", loc(1, 16)),
+		(ltok::COMMA, void, loc(1, 24)),
+		(ltok::LIT_STR, "return", loc(1, 25)),
+		(ltok::COMMA, void, loc(1, 34)),
+		(ltok::LIT_STR, "foo", loc(1, 35)),
 	];
 	lextest(in, expected);
 };
diff --git a/hare/lex/lex.ha b/hare/lex/lex.ha
@@ -189,6 +189,20 @@ fn lex_string(lex: *lexer, loc: location) (token | error) = {
 				strio::appendrune(buf, r);
 			},
 	};
+	match (nextw(lex)?) {
+		_: io::EOF => void,
+		r: (rune, location) => {
+			const r = r.0;
+			if (r == '"') {
+				const tok = lex_string(lex, loc)?;
+				const next = tok.1 as str;
+				strio::concat(buf, next);
+				free(next);
+			} else {
+				unget(lex, r);
+			};
+		},
+	};
 	return (ltok::LIT_STR, strio::finish(buf), loc);
 };