hare

commit 35d990514e1bc4ccd36f93d0179f341faf90df3a
parent 2b21998d1effb3bea66e7950b6d0732d73328318
Author: Drew DeVault <sir@cmpwn.com>
Date:   Wed, 14 Apr 2021 12:25:36 -0400

hare::lex: change approach to comments

Diffstat:
M
hare/lex/+test.ha
 | 
12
+++++-------
M
hare/lex/lex.ha
 | 
15
++++++++++++++-

2 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/hare/lex/+test.ha b/hare/lex/+test.ha
@@ -168,18 +168,16 @@ fn loc(line: uint, col: uint) location = location {
 	];
 	lextest(in, expected);
 
-	let in = "hello world // foo\n// bar";
+	let in = "// foo\n// bar\nhello world// baz\n\n// bad\ntest";
 	let buf = bufio::fixed(strings::toutf8(in), mode::READ);
 	defer io::close(buf);
 	let lexer = init(buf, "<input>", flags::COMMENTS);
 	assert(lex(&lexer) is token);
+	assert(comment(&lexer) == " foo\n bar\n");
 	assert(lex(&lexer) is token);
-	let tok = lex(&lexer) as token;
-	assert(tok.0 == ltok::COMMENT);
-	assert(tok.1 as str == " foo\n");
-	let tok = lex(&lexer) as token;
-	assert(tok.0 == ltok::COMMENT);
-	assert(tok.1 as str == " bar");
+	assert(comment(&lexer) == "");
+	assert(lex(&lexer) is token);
+	assert(comment(&lexer) == " bad\n");
 };
 
 @test fn runes() void = {
diff --git a/hare/lex/lex.ha b/hare/lex/lex.ha
@@ -17,6 +17,7 @@ export type lexer = struct {
 	un: (token | void),
 	rb: [2](rune | io::EOF | void),
 	flags: flags,
+	comment: str,
 };
 
 // Flags which apply to this lexer
@@ -54,9 +55,14 @@ export fn init(in: *io::stream, path: str, flags: flags...) lexer = {
 		un = void,
 		rb = [void...],
 		flags = f,
+		comment = "",
 	};
 };
 
+// Returns the current value of the comment buffer, or empty string if unset (or
+// if [flags::COMMENTS] was not enabled for this lexer).
+export fn comment(lex: *lexer) str = lex.comment;
+
 // Returns the next token from the lexer.
 export fn lex(lex: *lexer) (token | error) = {
 	match (lex.un) {
@@ -252,6 +258,7 @@ fn lex_comment(lexr: *lexer, loc: location) (token | error) = {
 	};
 
 	let buf = strio::dynamic();
+	defer io::close(buf);
 	for (true) match (next(lexr)?) {
 		io::EOF => break,
 		r: rune => {
@@ -259,7 +266,10 @@ fn lex_comment(lexr: *lexer, loc: location) (token | error) = {
 			if (r == '\n') break;
 		},
 	};
-	return (ltok::COMMENT, strio::finish(buf), loc);
+	let new = strings::concat(lexr.comment, strio::string(buf));
+	free(lexr.comment);
+	lexr.comment = new;
+	return lex(lexr);
 };
 
 fn lex2(lexr: *lexer, loc: location, r: rune) (token | error) = {
@@ -485,6 +495,9 @@ fn nextw(lex: *lexer) ((rune, location) | io::EOF | io::error) = {
 			e: (io::error | io::EOF) => return e,
 			r: rune => if (!ascii::isspace(r)) {
 				return (r, loc);
+			} else {
+				free(lex.comment);
+				lex.comment = "";
 			},
 		};
 	};