hare

The Hare programming language
git clone https://git.torresjrjr.com/hare.git
Log | Files | Refs | README | LICENSE

commit bf6029d3dc85a27873063a105c19265b599908ce
parent ecf1617e897fd290a5a8ea5e31c870c39c779faf
Author: Eyal Sawady <ecs@d2evs.net>
Date:   Sun, 11 Apr 2021 12:18:35 -0400

Refactor hare::lex and hare::parse

This does a lot of things which all necessarily had to happen at once:
- Move lex::location into lex::token
- Move lex::name and lex::literal into lex::ltok, and add lex::value for
  storing the value associated with them
- Replace io::EOF with ltok::EOF
- Drop want_name et al, and return lex::token rather than ltok from
  want_tok et al
- Use want_tok et al more pervasively now that they return lex::token

Diffstat:
Mhare/ast/expr.ha | 4++--
Mhare/lex/+test.ha | 265++++++++++++++++++++++++++++++++++++-------------------------------------------
Mhare/lex/lex.ha | 223++++++++++++++++++++++++++++++++++---------------------------------------------
Mhare/lex/token.ha | 75+++++++++++++++++++++++++++++++++++++++++----------------------------------
Mhare/parse/+test.ha | 20++++++++++++--------
Mhare/parse/decl.ha | 125++++++++++++++++++++++++++++++++-----------------------------------------------
Mhare/parse/expr.ha | 246++++++++++++++++++++++++++++++++++---------------------------------------------
Mhare/parse/exprclass.ha | 24+-----------------------
Mhare/parse/ident.ha | 10+++++-----
Mhare/parse/import.ha | 26+++++++++++++-------------
Mhare/parse/type.ha | 197++++++++++++++++++++++++++++++++++++-------------------------------------------
Mhare/parse/util.ha | 134+++++++++++++++++++++----------------------------------------------------------
12 files changed, 572 insertions(+), 777 deletions(-)

diff --git a/hare/ast/expr.ha b/hare/ast/expr.ha @@ -138,7 +138,7 @@ export type struct_constant = struct { export type tuple_constant = []*expr; // A constant -export type constant_expr = (void | lex::literal | array_constant | +export type constant_expr = (void | lex::value | array_constant | struct_constant | tuple_constant); // continue :label @@ -318,7 +318,7 @@ export fn expr_free(e: (expr | nullable *expr)) void = match (e) { expr_free(c._type); }, c: constant_expr => match(c) { - (void | lex::literal) => void, + (void | lex::value) => void, a: array_constant => { for (let i = 0z; i < len(a.values); i += 1) { expr_free(a.values[i]); diff --git a/hare/lex/+test.ha b/hare/lex/+test.ha @@ -19,19 +19,19 @@ use strings; @test fn unlex() void = { let lexer = init(io::empty, "<test>"); - unlex(&lexer, (btoken::IF, location { + unlex(&lexer, (ltok::IF, void, location { path = "<test>", line = 1234, col = 1234, })); - let t = lex(&lexer) as (token, location); - assert(t.0 is btoken); - assert(t.0 as btoken == btoken::IF); - assert(t.1.path == "<test>"); - assert(t.1.line == 1234 && t.1.col == 1234); + let t = lex(&lexer) as token; + assert(t.0 == ltok::IF); + assert(t.1 is void); + assert(t.2.path == "<test>"); + assert(t.2.line == 1234 && t.2.col == 1234); }; -fn litassert(expected: literal, actual: literal) void = match (expected) { +fn vassert(expected: value, actual: value) void = match (expected) { e: u8 => assert(actual as u8 == e), e: u16 => assert(actual as u16 == e), e: u32 => assert(actual as u32 == e), @@ -49,68 +49,48 @@ fn litassert(expected: literal, actual: literal) void = match (expected) { e: fconst => assert(actual as fconst == e), e: rune => assert(actual as rune == e), e: str => assert(actual as str == e), + e: void => assert(actual is void), }; -fn lextest(in: str, expected: [](uint, uint, token)) void = { +fn lextest(in: str, expected: []token) void = { let buf = bufio::fixed(strings::toutf8(in), mode::READ); let lexer = init(buf, "<test>"); for (let i = 0z; i < len(expected); i += 1) { - let eline = expected[i].0, ecol = expected[i].1, - etok = expected[i].2; + let etok = expected[i]; let tl = match (lex(&lexer)) { - tl: (token, location) => tl, - io::EOF => { - fmt::errorfln("unexpected EOF at {}", i); - abort(); - }, + tl: token => tl, err: error => { fmt::errorfln("{}: {}", i, strerror(err)); abort(); }, }; - let tok = tl.0, loc = tl.1; - match (tok) { - b: btoken => if (!(etok is btoken) || etok as btoken != b) { - fmt::errorfln("bad token at {}: got {}, wanted {}", - i, tokstr(tok), tokstr(etok)); - abort(); - }, - n: name => if (!(etok is name) || etok as name != n) { - fmt::errorfln("bad token at {}: got {}, wanted {}", - i, tokstr(tok), tokstr(etok)); - abort(); - }, - l: literal => if (!(etok is literal)) { - fmt::errorfln("bad token at {}: got {}, wanted {}", - i, tokstr(tok), tokstr(etok)); - abort(); - } else { - litassert(l, etok as literal); - }, - * => abort("TODO"), - }; - assert(loc.path == "<test>"); - if (loc.line != eline || loc.col != ecol) { - fmt::errorfln("bad line/col at {}: got {},{}; wanted {},{}", - i, loc.line, loc.col, eline, ecol); - abort(); - }; + assert(tl.0 == etok.0); + vassert(tl.1, etok.1); + assert(tl.2.line == etok.2.line && tl.2.col == etok.2.col + && tl.2.path == etok.2.path); }; - assert(lex(&lexer) is io::EOF); + let t = lex(&lexer) as token; + assert(t.0 == ltok::EOF); +}; + +fn loc(line: uint, col: uint) location = location { + path = "<test>", + line = line, + col = col, }; @test fn lex1() void = { const in = "~,{[(}]);"; - const expected: [_](uint, uint, token) = [ - (1, 1, btoken::BNOT), - (1, 2, btoken::COMMA), - (1, 3, btoken::LBRACE), - (1, 4, btoken::LBRACKET), - (1, 5, btoken::LPAREN), - (1, 6, btoken::RBRACE), - (1, 7, btoken::RBRACKET), - (1, 8, btoken::RPAREN), - (1, 9, btoken::SEMICOLON), + const expected: [_]token = [ + (ltok::BNOT, void, loc(1, 1)), + (ltok::COMMA, void, loc(1, 2)), + (ltok::LBRACE, void, loc(1, 3)), + (ltok::LBRACKET, void, loc(1, 4)), + (ltok::LPAREN, void, loc(1, 5)), + (ltok::RBRACE, void, loc(1, 6)), + (ltok::RBRACKET, void, loc(1, 7)), + (ltok::RPAREN, void, loc(1, 8)), + (ltok::SEMICOLON, void, loc(1, 9)), ]; lextest(in, expected); }; @@ -118,88 +98,83 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = { @test fn lex2() void = { // Ends with = to test =, EOF const in = "^ ^^ ^= * *= % %= + += - -= : :: & && &= | || |= = == / /= ="; - const expected: [_](uint, uint, token) = [ - (1, 1, btoken::BXOR), - (1, 3, btoken::LXOR), - (1, 6, btoken::BXOREQ), - (1, 9, btoken::TIMES), - (1, 11, btoken::TIMESEQ), - (1, 14, btoken::MODULO), - (1, 16, btoken::MODEQ), - (1, 19, btoken::PLUS), - (1, 21, btoken::PLUSEQ), - (1, 24, btoken::MINUS), - (1, 26, btoken::MINUSEQ), - (1, 29, btoken::COLON), - (1, 31, btoken::DOUBLE_COLON), - (1, 34, btoken::BAND), - (1, 36, btoken::LAND), - (1, 39, btoken::ANDEQ), - (1, 42, btoken::BOR), - (1, 44, btoken::LOR), - (1, 47, btoken::OREQ), - (1, 50, btoken::EQUAL), - (1, 52, btoken::LEQUAL), - (1, 55, btoken::DIV), - (1, 57, btoken::DIVEQ), - (1, 60, btoken::EQUAL), + const expected: [_]token = [ + (ltok::BXOR, void, loc(1, 1)), + (ltok::LXOR, void, loc(1, 3)), + (ltok::BXOREQ, void, loc(1, 6)), + (ltok::TIMES, void, loc(1, 9)), + (ltok::TIMESEQ, void, loc(1, 11)), + (ltok::MODULO, void, loc(1, 14)), + (ltok::MODEQ, void, loc(1, 16)), + (ltok::PLUS, void, loc(1, 19)), + (ltok::PLUSEQ, void, loc(1, 21)), + (ltok::MINUS, void, loc(1, 24)), + (ltok::MINUSEQ, void, loc(1, 26)), + (ltok::COLON, void, loc(1, 29)), + (ltok::DOUBLE_COLON, void, loc(1, 31)), + (ltok::BAND, void, loc(1, 34)), + (ltok::LAND, void, loc(1, 36)), + (ltok::ANDEQ, void, loc(1, 39)), + (ltok::BOR, void, loc(1, 42)), + (ltok::LOR, void, loc(1, 44)), + (ltok::OREQ, void, loc(1, 47)), + (ltok::EQUAL, void, loc(1, 50)), + (ltok::LEQUAL, void, loc(1, 52)), + (ltok::DIV, void, loc(1, 55)), + (ltok::DIVEQ, void, loc(1, 57)), + (ltok::EQUAL, void, loc(1, 60)), ]; lextest(in, expected); }; @test fn lex3() void = { const in = ". .. ... < << <= <<= > >> >= >>= >>"; - const expected: [_](uint, uint, token) = [ - (1, 1, btoken::DOT), - (1, 3, btoken::SLICE), - (1, 6, btoken::ELLIPSIS), - (1, 10, btoken::LESS), - (1, 12, btoken::LSHIFT), - (1, 15, btoken::LESSEQ), - (1, 18, btoken::LSHIFTEQ), - (1, 22, btoken::GREATER), - (1, 24, btoken::RSHIFT), - (1, 27, btoken::GREATEREQ), - (1, 30, btoken::RSHIFTEQ), - (1, 34, btoken::RSHIFT), + const expected: [_]token = [ + (ltok::DOT, void, loc(1, 1)), + (ltok::SLICE, void, loc(1, 3)), + (ltok::ELLIPSIS, void, loc(1, 6)), + (ltok::LESS, void, loc(1, 10)), + (ltok::LSHIFT, void, loc(1, 12)), + (ltok::LESSEQ, void, loc(1, 15)), + (ltok::LSHIFTEQ, void, loc(1, 18)), + (ltok::GREATER, void, loc(1, 22)), + (ltok::RSHIFT, void, loc(1, 24)), + (ltok::GREATEREQ, void, loc(1, 27)), + (ltok::RSHIFTEQ, void, loc(1, 30)), + (ltok::RSHIFT, void, loc(1, 34)), ]; lextest(in, expected); }; @test fn lexname() void = { const in = "hello world return void foobar"; - const expected: [_](uint, uint, token) = [ - (1, 1, "hello": name), - (1, 7, "world": name), - (1, 13, btoken::RETURN), - (1, 20, btoken::VOID), - (1, 25, "foobar": name), + const expected: [_]token = [ + (ltok::NAME, "hello", loc(1, 1)), + (ltok::NAME, "world", loc(1, 7)), + (ltok::RETURN, void, loc(1, 13)), + (ltok::VOID, void, loc(1, 20)), + (ltok::NAME, "foobar", loc(1, 25)), ]; lextest(in, expected); }; @test fn keywords() void = { - let keywords = bmap[..btoken::LAST_KEYWORD+1]; + let keywords = bmap[..ltok::LAST_KEYWORD+1]; for (let i = 0z; i < len(keywords); i += 1) { let lexer = init(bufio::fixed( strings::toutf8(keywords[i]), mode::READ), "<test>"); - let tl = match (lex(&lexer)) { - tl: (token, location) => tl, - * => abort(), - }; - let tok = tl.0; - assert(tok is btoken); - assert(tok as btoken == i: btoken); + let tok = lex(&lexer) as token; + assert(tok.0 == i: ltok); }; }; @test fn comments() void = { const in = "hello world // foo\nbar"; - const expected: [_](uint, uint, token) = [ - (1, 1, "hello": name), - (1, 7, "world": name), - (2, 1, "bar": name), + const expected: [_]token = [ + (ltok::NAME, "hello", loc(1, 1)), + (ltok::NAME, "world", loc(1, 7)), + (ltok::NAME, "bar", loc(2, 1)), ]; lextest(in, expected); }; @@ -207,22 +182,22 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = { @test fn runes() void = { const in = "'a' 'b' '\\a' '\\b' '\\f' '\\n' '\\r' '\\t' '\\v' '\\0' " "'\\\\' '\\\'' '\\x0A' '\\u1234' '\\U12345678'"; - const expected: [_](uint, uint, token) = [ - (1, 1, 'a'), - (1, 5, 'b'), - (1, 9, '\a'), - (1, 14, '\b'), - (1, 19, '\f'), - (1, 24, '\n'), - (1, 29, '\r'), - (1, 34, '\t'), - (1, 39, '\v'), - (1, 44, '\0'), - (1, 49, '\\'), - (1, 54, '\''), - (1, 59, '\x0A'), - (1, 66, '\u1234'), - (1, 75, '\U12345678'), + const expected: [_]token = [ + (ltok::LITERAL, 'a', loc(1, 1)), + (ltok::LITERAL, 'b', loc(1, 5)), + (ltok::LITERAL, '\a', loc(1, 9)), + (ltok::LITERAL, '\b', loc(1, 14)), + (ltok::LITERAL, '\f', loc(1, 19)), + (ltok::LITERAL, '\n', loc(1, 24)), + (ltok::LITERAL, '\r', loc(1, 29)), + (ltok::LITERAL, '\t', loc(1, 34)), + (ltok::LITERAL, '\v', loc(1, 39)), + (ltok::LITERAL, '\0', loc(1, 44)), + (ltok::LITERAL, '\\', loc(1, 49)), + (ltok::LITERAL, '\'', loc(1, 54)), + (ltok::LITERAL, '\x0A', loc(1, 59)), + (ltok::LITERAL, '\u1234', loc(1, 66)), + (ltok::LITERAL, '\U12345678', loc(1, 75)), ]; lextest(in, expected); }; @@ -230,33 +205,33 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = { @test fn strings() void = { const in = "\"a\" \"b\" \"\\a\" \"\\b\" \"\\f\" \"\\n\" \"\\r\" " "\"\\t\" \"\\v\" \"\\0\" \"\\\\\" \"\\\'\""; - const expected: [_](uint, uint, token) = [ - (1, 1, "a": literal), - (1, 5, "b": literal), - (1, 9, "\a": literal), - (1, 14, "\b": literal), - (1, 19, "\f": literal), - (1, 24, "\n": literal), - (1, 29, "\r": literal), - (1, 34, "\t": literal), - (1, 39, "\v": literal), - (1, 44, "\0": literal), - (1, 49, "\\": literal), - (1, 54, "\'": literal), + const expected: [_]token = [ + (ltok::LITERAL, "a", loc(1, 1)), + (ltok::LITERAL, "b", loc(1, 5)), + (ltok::LITERAL, "\a", loc(1, 9)), + (ltok::LITERAL, "\b", loc(1, 14)), + (ltok::LITERAL, "\f", loc(1, 19)), + (ltok::LITERAL, "\n", loc(1, 24)), + (ltok::LITERAL, "\r", loc(1, 29)), + (ltok::LITERAL, "\t", loc(1, 34)), + (ltok::LITERAL, "\v", loc(1, 39)), + (ltok::LITERAL, "\0", loc(1, 44)), + (ltok::LITERAL, "\\", loc(1, 49)), + (ltok::LITERAL, "\'", loc(1, 54)), ]; // TODO: test \x and \u and \U lextest(in, expected); const in = "\"ab\\a\\b\\f\\n\\r\\t\\v\\0\\\\\\'\""; - const expected: [_](uint, uint, token) = [ - (1, 1, "ab\a\b\f\n\r\t\v\0\\\'": literal), + const expected: [_]token = [ + (ltok::LITERAL, "ab\a\b\f\n\r\t\v\0\\\'", loc(1, 1)), ]; lextest(in, expected); const in = "\"hello world\" \"こんにちは\" \"return\" \"foo\""; - const expected: [_](uint, uint, token) = [ - (1, 1, "hello world": literal), - (1, 15, "こんにちは": literal), - (1, 23, "return": literal), - (1, 32, "foo": literal), + const expected: [_]token = [ + (ltok::LITERAL, "hello world", loc(1, 1)), + (ltok::LITERAL, "こんにちは", loc(1, 15)), + (ltok::LITERAL, "return", loc(1, 23)), + (ltok::LITERAL, "foo", loc(1, 32)), ]; lextest(in, expected); }; diff --git a/hare/lex/lex.ha b/hare/lex/lex.ha @@ -13,7 +13,7 @@ export type lexer = struct { in: *io::stream, path: str, loc: (uint, uint), - un: ((token, location) | void), + un: (token | void), rb: [2](rune | io::EOF | void), }; @@ -43,9 +43,9 @@ export fn init(in: *io::stream, path: str) lexer = lexer { }; // Returns the next token from the lexer. -export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = { +export fn lex(lex: *lexer) (token | error) = { match (lex.un) { - tok: (token, location) => { + tok: token => { lex.un = void; return tok; }, @@ -54,7 +54,7 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = { let loc = location { ... }; let r: rune = match (nextw(lex)?) { - io::EOF => return io::EOF, + io::EOF => return (ltok::EOF, void, mkloc(lex)), r: (rune, location) => { loc = r.1; r.0; @@ -70,7 +70,7 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = { abort(); // TODO: Literals }; - let tok: token = switch (r) { + let tok: ltok = switch (r) { * => return syntaxerr(loc, "invalid character"), '"', '\'' => { unget(lex, r); @@ -80,18 +80,18 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = { '^', '*', '%', '/', '+', '-', ':', '!', '&', '|', '=' => { return lex2(lex, loc, r); }, - '~' => btoken::BNOT, - ',' => btoken::COMMA, - '{' => btoken::LBRACE, - '[' => btoken::LBRACKET, - '(' => btoken::LPAREN, - '}' => btoken::RBRACE, - ']' => btoken::RBRACKET, - ')' => btoken::RPAREN, - ';' => btoken::SEMICOLON, - '?' => btoken::QUESTION, + '~' => ltok::BNOT, + ',' => ltok::COMMA, + '{' => ltok::LBRACE, + '[' => ltok::LBRACKET, + '(' => ltok::LPAREN, + '}' => ltok::RBRACE, + ']' => ltok::RBRACKET, + ')' => ltok::RPAREN, + ';' => ltok::SEMICOLON, + '?' => ltok::QUESTION, }; - return (tok, loc); + return (tok, void, loc); }; fn is_name(r: rune, num: bool) bool = @@ -156,10 +156,7 @@ fn lex_rune(lex: *lexer, loc: location) (rune | error) = { }; }; -fn lex_string( - lex: *lexer, - loc: location, -) ((token, location) | io::EOF | error) = { +fn lex_string(lex: *lexer, loc: location) (token | error) = { let chars: []u8 = []; for (true) match (next(lex)?) { io::EOF => return syntaxerr(loc, "unexpected EOF scanning string literal"), @@ -171,13 +168,10 @@ fn lex_string( append(chars, utf8::encoderune(r)...); }, }; - return (strings::fromutf8(chars): literal, loc); + return (ltok::LITERAL, strings::fromutf8(chars), loc); }; -fn lex_rn_str( - lex: *lexer, - loc: location, -) ((token, location) | io::EOF | error) = { +fn lex_rn_str(lex: *lexer, loc: location) (token | error) = { let r = match (next(lex)) { r: rune => r, (io::EOF | io::error) => abort(), @@ -189,7 +183,7 @@ fn lex_rn_str( }; // Rune literal - let ret: (token, location) = (lex_rune(lex, loc)?: literal, loc); + let ret: token = (ltok::LITERAL, lex_rune(lex, loc)?, loc); match (next(lex)?) { io::EOF => return syntaxerr(loc, "unexpected EOF"), @@ -199,10 +193,7 @@ fn lex_rn_str( return ret; }; -fn lex_name( - lex: *lexer, - loc: location, -) ((token, location) | io::EOF | error) = { +fn lex_name(lex: *lexer, loc: location) (token | error) = { let chars: []u8 = []; match (next(lex)) { r: rune => { @@ -224,46 +215,39 @@ fn lex_name( }; let n = strings::fromutf8(chars); - return match (sort::search(bmap[..btoken::LAST_KEYWORD+1], + return match (sort::search(bmap[..ltok::LAST_KEYWORD+1], size(str), &n, &ncmp)) { // TODO: Validate that names are ASCII - null => (n: name: token, loc), + null => (ltok::NAME, n, loc), v: *void => { let tok = v: uintptr - &bmap[0]: uintptr; tok /= size(str): uintptr; - (tok: btoken: token, loc); + (tok: ltok, void, loc); }, }; }; -fn lex2( - lexr: *lexer, - loc: location, - r: rune, -) ((token, location) | io::EOF | error) = { - let n = match (next(lexr)?) { - io::EOF => io::EOF, - r: rune => r, - }; - let tok: token = switch (r) { +fn lex2(lexr: *lexer, loc: location, r: rune) (token | error) = { + let n = next(lexr)?; + let tok: ltok = switch (r) { '^' => match (n) { r: rune => switch (r) { - '^' => return (btoken::LXOR: token, loc), - '=' => return (btoken::BXOREQ: token, loc), - * => btoken::BXOR, + '^' => return (ltok::LXOR, void, loc), + '=' => return (ltok::BXOREQ, void, loc), + * => ltok::BXOR, }, - io::EOF => btoken::BXOR, + io::EOF => ltok::BXOR, }, '*' => match (n) { r: rune => switch (r) { - '=' => return (btoken::TIMESEQ: token, loc), - * => btoken::TIMES, + '=' => return (ltok::TIMESEQ, void, loc), + * => ltok::TIMES, }, - io::EOF => btoken::TIMES, + io::EOF => ltok::TIMES, }, '/' => match (n) { r: rune => switch (r) { - '=' => return (btoken::DIVEQ: token, loc), + '=' => return (ltok::DIVEQ, void, loc), '/' => { // Comment for (true) match (next(lexr)?) { @@ -274,84 +258,81 @@ fn lex2( }; return lex(lexr); }, - * => btoken::DIV, + * => ltok::DIV, }, - io::EOF => btoken::DIV, + io::EOF => ltok::DIV, }, '%' => match (n) { r: rune => switch (r) { - '=' => return (btoken::MODEQ: token, loc), - * => btoken::MODULO, + '=' => return (ltok::MODEQ, void, loc), + * => ltok::MODULO, }, - io::EOF => btoken::MODULO, + io::EOF => ltok::MODULO, }, '+' => match (n) { r: rune => switch (r) { - '=' => return (btoken::PLUSEQ: token, loc), - * => btoken::PLUS, + '=' => return (ltok::PLUSEQ, void, loc), + * => ltok::PLUS, }, - io::EOF => btoken::PLUS, + io::EOF => ltok::PLUS, }, '-' => match (n) { r: rune => switch (r) { - '=' => return (btoken::MINUSEQ: token, loc), - * => btoken::MINUS, + '=' => return (ltok::MINUSEQ, void, loc), + * => ltok::MINUS, }, - io::EOF => btoken::MINUS, + io::EOF => ltok::MINUS, }, ':' => match (n) { r: rune => switch (r) { - ':' => return (btoken::DOUBLE_COLON: token, loc), - * => btoken::COLON, + ':' => return (ltok::DOUBLE_COLON, void, loc), + * => ltok::COLON, }, - io::EOF => btoken::COLON, + io::EOF => ltok::COLON, }, '!' => match (n) { r: rune => switch (r) { - '=' => return (btoken::NEQUAL: token, loc), - * => btoken::LNOT, + '=' => return (ltok::NEQUAL, void, loc), + * => ltok::LNOT, }, - io::EOF => btoken::LNOT, + io::EOF => ltok::LNOT, }, '&' => match (n) { r: rune => switch (r) { - '&' => return (btoken::LAND: token, loc), - '=' => return (btoken::ANDEQ: token, loc), - * => btoken::BAND, + '&' => return (ltok::LAND, void, loc), + '=' => return (ltok::ANDEQ, void, loc), + * => ltok::BAND, }, - io::EOF => btoken::BAND, + io::EOF => ltok::BAND, }, '|' => match (n) { r: rune => switch (r) { - '|' => return (btoken::LOR: token, loc), - '=' => return (btoken::OREQ: token, loc), - * => btoken::BOR, + '|' => return (ltok::LOR, void, loc), + '=' => return (ltok::OREQ, void, loc), + * => ltok::BOR, }, - io::EOF => btoken::BOR, + io::EOF => ltok::BOR, }, '=' => match (n) { r: rune => switch (r) { - '=' => return (btoken::LEQUAL: token, loc), - * => btoken::EQUAL, + '=' => return (ltok::LEQUAL, void, loc), + * => ltok::EQUAL, }, - io::EOF => btoken::EQUAL, + io::EOF => ltok::EQUAL, }, * => return syntaxerr(loc, "unknown token sequence"), }; unget(lexr, n); - return (tok, loc); + return (tok, void, loc); }; -fn lex3( - lex: *lexer, - loc: location, - r: rune, -) ((token, location) | io::EOF | error) = { +fn lex3(lex: *lexer, loc: location, r: rune) (token | error) = { let n = match (next(lex)?) { io::EOF => return switch (r) { - '.' => (btoken::DOT: token, loc), - '<' => (btoken::LESS: token, loc), - '>' => (btoken::GREATER: token, loc), + '.' => (ltok::DOT, void, loc), + '<' => (ltok::LESS, void, loc), + '>' => (ltok::GREATER, void, loc), + * => abort(), // Invariant }, r: rune => r, }; @@ -363,12 +344,8 @@ fn lex3( }; }; -fn lex3dot( - lex: *lexer, - loc: location, - n: rune, -) ((token, location) | io::EOF | error) = { - let tok: token = switch (n) { +fn lex3dot(lex: *lexer, loc: location, n: rune) (token | error) = { + let tok: ltok = switch (n) { '.' => { let q = match (next(lex)?) { io::EOF => io::EOF, @@ -376,28 +353,24 @@ fn lex3dot( }; let t = match (q) { r: rune => switch (r) { - '.' => return (btoken::ELLIPSIS: token, loc), - * => btoken::SLICE, + '.' => return (ltok::ELLIPSIS, void, loc), + * => ltok::SLICE, }, - io::EOF => btoken::SLICE, + io::EOF => ltok::SLICE, }; unget(lex, q); t; }, * => { unget(lex, n); - btoken::DOT; + ltok::DOT; } }; - return (tok, loc); + return (tok, void, loc); }; -fn lex3lt( - lex: *lexer, - loc: location, - n: rune, -) ((token, location) | io::EOF | error) = { - let tok: token = switch (n) { +fn lex3lt(lex: *lexer, loc: location, n: rune) (token | error) = { + let tok: ltok = switch (n) { '<' => { let q = match (next(lex)?) { io::EOF => io::EOF, @@ -405,29 +378,25 @@ fn lex3lt( }; let t = match (q) { r: rune => switch (r) { - '=' => return (btoken::LSHIFTEQ: token, loc), - * => btoken::LSHIFT, + '=' => return (ltok::LSHIFTEQ, void, loc), + * => ltok::LSHIFT, }, - io::EOF => btoken::LSHIFT, + io::EOF => ltok::LSHIFT, }; unget(lex, q); t; }, - '=' => btoken::LESSEQ, + '=' => ltok::LESSEQ, * => { unget(lex, n); - btoken::LESS; + ltok::LESS; } }; - return (tok, loc); + return (tok, void, loc); }; -fn lex3gt( - lex: *lexer, - loc: location, - n: rune, -) ((token, location) | io::EOF | error) = { - let tok: token = switch (n) { +fn lex3gt(lex: *lexer, loc: location, n: rune) (token | error) = { + let tok: ltok = switch (n) { '>' => { let q = match (next(lex)?) { io::EOF => io::EOF, @@ -435,27 +404,27 @@ fn lex3gt( }; let t = match (q) { r: rune => switch (r) { - '=' => return (btoken::RSHIFTEQ: token, loc), - * => btoken::RSHIFT, + '=' => return (ltok::RSHIFTEQ, void, loc), + * => ltok::RSHIFT, }, - io::EOF => btoken::RSHIFT, + io::EOF => ltok::RSHIFT, }; unget(lex, q); t; }, - '=' => btoken::GREATEREQ, + '=' => ltok::GREATEREQ, * => { unget(lex, n); - btoken::GREATER; + ltok::GREATER; } }; - return (tok, loc); + return (tok, void, loc); }; -// Unlex a single token. The next call to [lex] will return this token, location -// pair. Only one unlex is supported at a time; you must call [lex] before -// calling [unlex] again. -export fn unlex(lex: *lexer, tok: (token, location)) void = { +// Unlex a single token. The next call to [lex] will return this token. Only one +// unlex is supported at a time; you must call [lex] before calling [unlex] +// again. +export fn unlex(lex: *lexer, tok: token) void = { assert(lex.un is void, "attempted to unlex more than one token"); lex.un = tok; }; diff --git a/hare/lex/token.ha b/hare/lex/token.ha @@ -2,7 +2,7 @@ use encoding::utf8; use strings; // A token with no additional context, such as '+' -export type btoken = enum { +export type ltok = enum { // Keep ordered with bmap // Alpha sorted ATTR_FINI, @@ -117,10 +117,16 @@ export type btoken = enum { SLICE, TIMES, TIMESEQ, + LAST_BTOK = TIMESEQ, + + NAME, + LITERAL, + LABEL, + EOF, }; const bmap: [_]str = [ - // Keep ordered with btoken + // Keep ordered with tok "@fini", "@init", "@noreturn", @@ -232,18 +238,13 @@ const bmap: [_]str = [ "*=", ]; -// A loop label, such as ':example' -export type label = str; - -// A name, such as 'example' -export type name = str; - export type iconst = i64; export type fconst = f64; // A token for a literal value, such as '1337u32' -export type literal = (u8 | u16 | u32 | u64 | uint | uintptr | i8 | i16 | i32 | - i64 | int | iconst | f32 | f64 | fconst | rune | str); +// TODO: Refactor this into a union { i64, u64, f64, str } + storage enum +export type value = (u8 | u16 | u32 | u64 | uint | uintptr | i8 | i16 | i32 | + i64 | int | iconst | f32 | f64 | fconst | rune | str | void); // A location within a source file. // The path is borrowed from the file name given to the lexer. @@ -254,30 +255,36 @@ export type location = struct { }; // A single lexical token. -export type token = (btoken | label | name | literal); +export type token = (ltok, value, location); // Converts a token to its string representation -export fn tokstr(tok: token) const str = match (tok) { - b: btoken => bmap[b: int], - n: name => n: str, - l: literal => match (l) { - u8 => "u8", - u16 => "u16", - u32 => "u32", - u64 => "u64", - uint => "uint", - uintptr => "uintptr", - i8 => "i8", - i16 => "i16", - i32 => "i32", - i64 => "i64", - int => "int", - iconst => "iconst", - f32 => "f32", - f64 => "f64", - fconst => "fconst", - rune => "rune", - str => "str", - }, - * => abort(), // TODO +export fn tokstr(tok: token) const str = { + if (tok.0 <= ltok::LAST_BTOK) { + return bmap[tok.0: int]; + }; + return switch (tok.0) { + ltok::NAME => tok.1 as str, + ltok::LABEL => abort(), // TODO + ltok::LITERAL => match (tok.1) { + u8 => "u8", + u16 => "u16", + u32 => "u32", + u64 => "u64", + uint => "uint", + uintptr => "uintptr", + i8 => "i8", + i16 => "i16", + i32 => "i32", + i64 => "i64", + int => "int", + iconst => "iconst", + f32 => "f32", + f64 => "f64", + fconst => "fconst", + rune => "rune", + str => "str", + }, + ltok::EOF => "EOF", + * => abort(), + }; }; diff --git a/hare/parse/+test.ha b/hare/parse/+test.ha @@ -16,7 +16,8 @@ use strio; defer ast::ident_free(ident); assert(len(ident) == 1); assert(ident[0] == "foo"); - assert(lex::lex(&lexer) is io::EOF); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::EOF); }; { @@ -27,7 +28,8 @@ use strio; defer ast::ident_free(ident); assert(len(ident) == 2); assert(ident[0] == "foo" && ident[1] == "bar"); - assert(lex::lex(&lexer) is io::EOF); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::EOF); }; { @@ -39,7 +41,8 @@ use strio; assert(len(ident) == 3); assert(ident[0] == "foo" && ident[1] == "bar" && ident[2] == "baz"); - assert(lex::lex(&lexer) is io::EOF); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::EOF); }; { @@ -50,8 +53,8 @@ use strio; defer ast::ident_free(ident); assert(len(ident) == 2); assert(ident[0] == "foo" && ident[1] == "bar"); - let tok = lex::lex(&lexer) as (lex::token, lex::location); - assert(tok.0 as lex::btoken == lex::btoken::SEMICOLON); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::SEMICOLON); }; }; @@ -70,7 +73,8 @@ use strio; let mod = mods[0] as ast::import_module; assert(len(mod) == 1 && mod[0] == "foo"); - assert(lex::lex(&lexer) is io::EOF); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::EOF); }; { @@ -98,8 +102,8 @@ use strio; }; }; - let tok = lex::lex(&lexer) as (lex::token, lex::location); - assert(tok.0 as lex::btoken == lex::btoken::EXPORT); + let tok = lex::lex(&lexer) as lex::token; + assert(tok.0 == lex::ltok::EXPORT); }; { diff --git a/hare/parse/decl.ha b/hare/parse/decl.ha @@ -1,67 +1,59 @@ use ascii; use hare::ast; -use hare::lex::{btoken}; +use hare::lex::{ltok}; use hare::lex; use hare::unparse; use strings; fn attr_symbol(lexer: *lex::lexer) (str | error) = { - want_btoken(lexer, btoken::LPAREN)?; - let s: (str, lex::location) = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Expected string, got EOF"), - t: (lex::token, lex::location) => match (t.0) { - l: lex::literal => match (l) { - s: str => (s, t.1), - * => return syntaxerr(t.1, - "Unexpected {}, was expecting string", - lex::tokstr(t.0)), - }, - * => return syntaxerr(t.1, - "Unexpected {}, was expecting string", - lex::tokstr(t.0)), - }, + want_tok(lexer, ltok::LPAREN)?; + let t = want_tok(lexer, ltok::LITERAL)?; + let s = match (t.1) { + s: str => s, + * => return syntaxerr(t.2, + "Unexpected {}, was expecting string", + lex::tokstr(t)), }; - let d = strings::iter(s.0); + let d = strings::iter(s); match (strings::next(&d)) { void => void, - r: rune => synassert(s.1, + r: rune => synassert(t.2, ascii::isalpha(r) || r == '.' || r == '_', "Invalid symbol")?, }; for (true) match (strings::next(&d)) { void => break, - r: rune => synassert(s.1, + r: rune => synassert(t.2, ascii::isalnum(r) || r == '$' || r == '.' || r == '_', "Invalid symbol")?, }; - want_btoken(lexer, btoken::RPAREN)?; - return s.0; + want_tok(lexer, ltok::RPAREN)?; + return s; }; fn decl_global( lexer: *lex::lexer, - tok: btoken + tok: ltok ) ([]ast::decl_global | error) = { let decl: []ast::decl_global = []; for (true) { - let symbol = if (tok == btoken::CONST || tok == btoken::LET) { - match (try_btoken(lexer, btoken::ATTR_SYMBOL)?) { + let symbol = if (tok == ltok::CONST || tok == ltok::LET) { + match (try_tok(lexer, ltok::ATTR_SYMBOL)?) { void => "", - lex::btoken => attr_symbol(lexer)?, + lex::token => attr_symbol(lexer)?, }; } else ""; let ident = ident(lexer)?; - want_btoken(lexer, btoken::COLON)?; + want_tok(lexer, ltok::COLON)?; let _type = _type(lexer)?; - if (tok == btoken::CONST) { + if (tok == ltok::CONST) { _type.flags |= ast::type_flags::CONST; }; - want_btoken(lexer, btoken::EQUAL)?; + want_tok(lexer, ltok::EQUAL)?; let init = expression(lexer)?; - let btok = try_btoken(lexer, btoken::COMMA)?; + let btok = try_tok(lexer, ltok::COMMA)?; append(decl, ast::decl_global { - is_const = tok == btoken::DEF, + is_const = tok == ltok::DEF, symbol = symbol, ident = ident, _type = _type, @@ -79,9 +71,9 @@ fn decl_type(lexer: *lex::lexer) ([]ast::decl_type | error) = { let decl: []ast::decl_type = []; for (true) { let ident = ident(lexer)?; - want_btoken(lexer, btoken::EQUAL)?; + want_tok(lexer, ltok::EQUAL)?; let _type = _type(lexer)?; - let btok = try_btoken(lexer, btoken::COMMA)?; + let btok = try_tok(lexer, ltok::COMMA)?; append(decl, ast::decl_type { ident = ident, _type = _type, @@ -96,23 +88,23 @@ fn decl_type(lexer: *lex::lexer) ([]ast::decl_type | error) = { fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = { let attr = ast::fndecl_attrs::NONE, noreturn = false, sym = ""; - let attrs = [ - btoken::ATTR_FINI, btoken::ATTR_INIT, btoken::ATTR_TEST, - btoken::ATTR_NORETURN, btoken::ATTR_SYMBOL + const attrs = [ + ltok::ATTR_FINI, ltok::ATTR_INIT, ltok::ATTR_TEST, + ltok::ATTR_NORETURN, ltok::ATTR_SYMBOL ]; - for (true) match (try_btoken(lexer, attrs...)?) { + for (true) match (try_tok(lexer, attrs...)?) { void => break, - b: lex::btoken => switch (b) { - btoken::ATTR_FINI => attr = ast::fndecl_attrs::FINI, - btoken::ATTR_INIT => attr = ast::fndecl_attrs::INIT, - btoken::ATTR_TEST => attr = ast::fndecl_attrs::TEST, - btoken::ATTR_NORETURN => noreturn = true, - btoken::ATTR_SYMBOL => sym = attr_symbol(lexer)?, + t: lex::token => switch (t.0) { + ltok::ATTR_FINI => attr = ast::fndecl_attrs::FINI, + ltok::ATTR_INIT => attr = ast::fndecl_attrs::INIT, + ltok::ATTR_TEST => attr = ast::fndecl_attrs::TEST, + ltok::ATTR_NORETURN => noreturn = true, + ltok::ATTR_SYMBOL => sym = attr_symbol(lexer)?, * => abort("unreachable"), }, }; - want_btoken(lexer, btoken::FN)?; + want_tok(lexer, ltok::FN)?; let ident_loc = mkloc(lexer); let ident = ident(lexer)?; let proto_loc = mkloc(lexer); @@ -121,9 +113,9 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = { prototype.attrs |= ast::func_attrs::NORETURN; }; - let tok = want_btoken(lexer, btoken::EQUAL, btoken::SEMICOLON)?; - let body = switch (tok) { - btoken::EQUAL => { + let tok = want_tok(lexer, ltok::EQUAL, ltok::SEMICOLON)?; + let body = switch (tok.0) { + ltok::EQUAL => { synassert(ident_loc, len(ident) == 1, "Unexpected identifier, was expecting name")?; const params = prototype.params; @@ -132,10 +124,9 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = { len(params[i].name) > 0, "Expected parameter name in function declaration")?; }; - compound_expression(lexer)?; + expression(lexer)?; }, - // We don't care about the location - btoken::SEMICOLON => lex::unlex(lexer, (tok, proto_loc)), + ltok::SEMICOLON => lex::unlex(lexer, tok), }; return ast::decl_func { @@ -155,40 +146,24 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = { export fn decls(lexer: *lex::lexer) ([]ast::decl | error) = { let decls: []ast::decl = []; for (true) { - match (lex::lex(lexer)?) { - io::EOF => break, - t: (lex::token, lex::location) => lex::unlex(lexer, t), - }; - let exported = match (try_btoken(lexer, btoken::EXPORT)?) { + if (peek_tok(lexer, ltok::EOF)? is lex::token) break; + let exported = match (try_tok(lexer, ltok::EXPORT)?) { void => false, - lex::btoken => true, - }; - let tok = match(lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Unexpected EOF, was expecting declaration"), - t: (lex::token, lex::location) => t, - }; - let btok = match (tok.0) { - b: lex::btoken => b, - * => return syntaxerr(mkloc(lexer), - "Unexpected {}, was expecting declaration", - lex::tokstr(tok.0)), + lex::token => true, }; - let decl = switch (btok) { - btoken::CONST, btoken::LET, btoken::DEF => - decl_global(lexer, btok)?, - btoken::TYPE => decl_type(lexer)?, - * => { - lex::unlex(lexer, tok); - decl_func(lexer)?; - }, + const toks = [ltok::CONST, ltok::LET, ltok::DEF, ltok::TYPE]; + let decl = match (try_tok(lexer, toks...)?) { + void => decl_func(lexer)?, + t: lex::token => + if (t.0 == ltok::TYPE) decl_type(lexer)? + else decl_global(lexer, t.0)?, }; append(decls, ast::decl { exported = exported, loc = mkloc(lexer), decl = decl, }); - want_btoken(lexer, btoken::SEMICOLON)?; + want_tok(lexer, ltok::SEMICOLON)?; }; return decls; }; diff --git a/hare/parse/expr.ha b/hare/parse/expr.ha @@ -1,5 +1,5 @@ use hare::ast; -use hare::lex::{btoken}; +use hare::lex::{ltok}; use hare::lex; fn binarithm( @@ -14,19 +14,19 @@ fn binarithm( expr: ast::expr => expr, }; - let tok = mustlex(lexer)?; + let tok = lex::lex(lexer)?; let j = precedence(tok); for (j >= i; j = precedence(tok)) { const op = binop_for_tok(tok); let rvalue = cast(lexer, void)?; - tok = mustlex(lexer)?; + tok = lex::lex(lexer)?; let k = precedence(tok); for (k > j; k = precedence(tok)) { lex::unlex(lexer, tok); rvalue = binarithm(lexer, rvalue, k)?; - tok = mustlex(lexer)?; + tok = lex::lex(lexer)?; }; let expr = ast::binarithm_expr { @@ -46,15 +46,15 @@ fn cast(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = { void => unarithm(lexer)?, e: ast::expr => e, }; - const tok = match (try_btoken(lexer, btoken::COLON, - btoken::AS, btoken::IS)?) { + const tok = match (try_tok(lexer, ltok::COLON, + ltok::AS, ltok::IS)?) { void => return lvalue, - tok: btoken => tok, + tok: lex::token => tok.0, }; const kind = switch (tok) { - btoken::COLON => ast::cast_kind::CAST, - btoken::AS => ast::cast_kind::ASSERTION, - btoken::IS => ast::cast_kind::TEST, + ltok::COLON => ast::cast_kind::CAST, + ltok::AS => ast::cast_kind::ASSERTION, + ltok::IS => ast::cast_kind::TEST, * => abort(), }; return cast(lexer, ast::cast_expr { @@ -65,7 +65,7 @@ fn cast(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = { }; fn constant(lexer: *lex::lexer) (ast::expr | error) = { - want_btoken(lexer, btoken::VOID)?; + want_tok(lexer, ltok::VOID)?; return void; }; @@ -78,45 +78,29 @@ fn objsel(lexer: *lex::lexer) (ast::expr | error) = { }; fn plain_expression(lexer: *lex::lexer) (ast::expr | error) = { - let tok = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Unexpected EOF, was expecting an expression"), - tok: (lex::token, lex::location) => tok, - }; - let tok = match (tok.0) { - btok: btoken => { - lex::unlex(lexer, tok); - btok; - }, - lit: lex::literal => { - lex::unlex(lexer, tok); - return constant(lexer); + let tok = peek_tok(lexer)? as lex::token; + return switch (tok.0) { + ltok::TRUE, + ltok::FALSE, + ltok::NULL, + ltok::VOID => constant(lexer), + ltok::LBRACKET => abort(), // TODO: Array literal + ltok::STRUCT => abort(), // TODO: Struct literal + ltok::LPAREN => { + let ex = expression(lexer); + return switch (want_tok(lexer, + ltok::RPAREN, ltok::COMMA)?.0) { + ltok::RPAREN => ex, + ltok::COMMA => abort(), // TODO: Tuple literal + * => abort(), + }; }, - name: lex::name => { - lex::unlex(lexer, tok); + ltok::LITERAL => return constant(lexer), + ltok::NAME => { let id = ident(lexer)?; - return match (try_btoken(lexer, btoken::LBRACE)?) { + return match (try_tok(lexer, ltok::LBRACE)?) { void => id: ast::access_identifier, - btoken => abort(), // TODO: Struct literal - }; - }, - }; - return switch (tok) { - btoken::TRUE, - btoken::FALSE, - btoken::NULL, - btoken::VOID => { - constant(lexer); - }, - btoken::LBRACKET => abort(), // TODO: Array literal - btoken::STRUCT => abort(), // TODO: Struct literal - btoken::LPAREN => { - let ex = expression(lexer); - return switch (want_btoken(lexer, - btoken::RPAREN, btoken::COMMA)?) { - btoken::RPAREN => ex, - btoken::COMMA => abort(), // TODO: Tuple literal - * => abort(), + lex::token => abort(), // TODO: Struct literal }; }, * => syntaxerr(mkloc(lexer), @@ -127,12 +111,12 @@ fn plain_expression(lexer: *lex::lexer) (ast::expr | error) = { fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = { // Built-ins (XXX: these should probably be moved in the Hare grammar) - match (peek_btoken(lexer, - btoken::ALLOC, btoken::APPEND, btoken::FREE, - btoken::DELETE, btoken::ABORT, btoken::ASSERT, - btoken::STATIC, btoken::SIZE, btoken::LEN, - btoken::OFFSET)?) { - tok: btoken => { + match (peek_tok(lexer, + ltok::ALLOC, ltok::APPEND, ltok::FREE, + ltok::DELETE, ltok::ABORT, ltok::ASSERT, + ltok::STATIC, ltok::SIZE, ltok::LEN, + ltok::OFFSET)?) { + tok: lex::token => { if (lvalue is ast::expr) { return syntaxerr(mkloc(lexer), "Unexpected {}, was expecting '(', '.', or '['", @@ -148,13 +132,13 @@ fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = ex: ast::expr => ex, }; - match (try_btoken(lexer, btoken::LPAREN, btoken::DOT, btoken::LBRACKET, - btoken::QUESTION)) { - tok: btoken => switch (tok) { - btoken::LPAREN => abort(), // TODO: Calls - btoken::DOT => abort(), // TODO: Field access - btoken::LBRACKET => abort(), // TODO: Indexing - btoken::QUESTION => abort(), // TODO: Propagation + match (try_tok(lexer, ltok::LPAREN, ltok::DOT, ltok::LBRACKET, + ltok::QUESTION)) { + tok: lex::token => switch (tok.0) { + ltok::LPAREN => abort(), // TODO: Calls + ltok::DOT => abort(), // TODO: Field access + ltok::LBRACKET => abort(), // TODO: Indexing + ltok::QUESTION => abort(), // TODO: Propagation * => abort(), }, void => return lvalue, @@ -164,23 +148,23 @@ fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = }; fn unarithm(lexer: *lex::lexer) (ast::expr | error) = { - const tok = match (try_btoken(lexer, - btoken::PLUS, btoken::MINUS, btoken::BNOT, - btoken::LNOT, btoken::TIMES, btoken::BAND)) { + const tok = match (try_tok(lexer, + ltok::PLUS, ltok::MINUS, ltok::BNOT, + ltok::LNOT, ltok::TIMES, ltok::BAND)) { void => return postfix(lexer, void), - tok: btoken => tok, + tok: lex::token => tok.0, }; const op = switch (tok) { - btoken::PLUS => ast::unarithm_op::PLUS, - btoken::MINUS => ast::unarithm_op::MINUS, - btoken::BNOT => ast::unarithm_op::BNOT, - btoken::LNOT => ast::unarithm_op::LNOT, - btoken::TIMES => ast::unarithm_op::DEREF, - btoken::BAND => ast::unarithm_op::ADDR, + ltok::PLUS => ast::unarithm_op::PLUS, + ltok::MINUS => ast::unarithm_op::MINUS, + ltok::BNOT => ast::unarithm_op::BNOT, + ltok::LNOT => ast::unarithm_op::LNOT, + ltok::TIMES => ast::unarithm_op::DEREF, + ltok::BAND => ast::unarithm_op::ADDR, * => abort(), }; let operand = - if (tok == btoken::BAND) objsel(lexer)? + if (tok == ltok::BAND) objsel(lexer)? else unarithm(lexer)?; return ast::unarithm_expr { op = op, @@ -188,87 +172,71 @@ fn unarithm(lexer: *lex::lexer) (ast::expr | error) = { }; }; -fn binop_for_tok(tok: (lex::token, lex::location)) ast::binarithm_op = { - let tok = match (tok.0) { - b: btoken => b, - * => abort(), - }; - return switch (tok) { - btoken::BAND => ast::binarithm_op::BAND, - btoken::BOR => ast::binarithm_op::BOR, - btoken::BXOR => ast::binarithm_op::BXOR, - btoken::DIV => ast::binarithm_op::DIV, - btoken::GREATER => ast::binarithm_op::GT, - btoken::GREATEREQ => ast::binarithm_op::GTEQ, - btoken::LAND => ast::binarithm_op::LAND, - btoken::LEQUAL => ast::binarithm_op::LEQUAL, - btoken::LESS => ast::binarithm_op::LESS, - btoken::LESSEQ => ast::binarithm_op::LESSEQ, - btoken::LOR => ast::binarithm_op::LOR, - btoken::LSHIFT => ast::binarithm_op::LSHIFT, - btoken::LXOR => ast::binarithm_op::LXOR, - btoken::MINUS => ast::binarithm_op::MINUS, - btoken::MODULO => ast::binarithm_op::MODULO, - btoken::NEQUAL => ast::binarithm_op::NEQUAL, - btoken::PLUS => ast::binarithm_op::PLUS, - btoken::RSHIFT => ast::binarithm_op::RSHIFT, - btoken::TIMES => ast::binarithm_op::TIMES, - * => abort(), - }; +fn binop_for_tok(tok: lex::token) ast::binarithm_op = switch (tok.0) { + ltok::BAND => ast::binarithm_op::BAND, + ltok::BOR => ast::binarithm_op::BOR, + ltok::BXOR => ast::binarithm_op::BXOR, + ltok::DIV => ast::binarithm_op::DIV, + ltok::GREATER => ast::binarithm_op::GT, + ltok::GREATEREQ => ast::binarithm_op::GTEQ, + ltok::LAND => ast::binarithm_op::LAND, + ltok::LEQUAL => ast::binarithm_op::LEQUAL, + ltok::LESS => ast::binarithm_op::LESS, + ltok::LESSEQ => ast::binarithm_op::LESSEQ, + ltok::LOR => ast::binarithm_op::LOR, + ltok::LSHIFT => ast::binarithm_op::LSHIFT, + ltok::LXOR => ast::binarithm_op::LXOR, + ltok::MINUS => ast::binarithm_op::MINUS, + ltok::MODULO => ast::binarithm_op::MODULO, + ltok::NEQUAL => ast::binarithm_op::NEQUAL, + ltok::PLUS => ast::binarithm_op::PLUS, + ltok::RSHIFT => ast::binarithm_op::RSHIFT, + ltok::TIMES => ast::binarithm_op::TIMES, + * => abort(), }; -fn precedence(tok: (lex::token, lex::location)) int = { - let tok = match (tok.0) { - b: btoken => b, - * => return -1, - }; - return switch (tok) { - btoken::LOR => 0, - btoken::LXOR => 1, - btoken::LAND => 2, - btoken::LEQUAL, - btoken::NEQUAL => 3, - btoken::LESS, - btoken::LESSEQ, - btoken::GREATER, - btoken::GREATEREQ => 4, - btoken::BOR => 5, - btoken::BXOR => 6, - btoken::BAND => 7, - btoken::LSHIFT, - btoken::RSHIFT => 8, - btoken::PLUS, - btoken::MINUS => 9, - btoken::TIMES, - btoken::DIV, - btoken::MODULO => 10, - * => -1, - }; +fn precedence(tok: lex::token) int = switch (tok.0) { + ltok::LOR => 0, + ltok::LXOR => 1, + ltok::LAND => 2, + ltok::LEQUAL, + ltok::NEQUAL => 3, + ltok::LESS, + ltok::LESSEQ, + ltok::GREATER, + ltok::GREATEREQ => 4, + ltok::BOR => 5, + ltok::BXOR => 6, + ltok::BAND => 7, + ltok::LSHIFT, + ltok::RSHIFT => 8, + ltok::PLUS, + ltok::MINUS => 9, + ltok::TIMES, + ltok::DIV, + ltok::MODULO => 10, + * => -1, }; fn expression_list(lexer: *lex::lexer) (ast::expr | error) = { let items: ast::list_expr = []; for (let more = true; more) { - let tok = mustlex(lexer)?; - lex::unlex(lexer, tok); - let item = match (tok.0) { - tok: btoken => switch (tok) { - btoken::RBRACE => break, - btoken::BREAK, - btoken::CONTINUE, - btoken::RETURN => { - more = false; - control_statement(lexer)?; - }, - * => expression(lexer)?, + let tok = peek_tok(lexer)? as lex::token; + let item = switch (tok.0) { + ltok::RBRACE => break, + ltok::BREAK, + ltok::CONTINUE, + ltok::RETURN => { + more = false; + control_statement(lexer)?; }, * => expression(lexer)?, }; append(items, alloc(item)); - want_btoken(lexer, btoken::SEMICOLON)?; + want_tok(lexer, ltok::SEMICOLON)?; }; - want_btoken(lexer, btoken::RBRACE)?; + want_tok(lexer, ltok::RBRACE)?; return items; }; diff --git a/hare/parse/exprclass.ha b/hare/parse/exprclass.ha @@ -1,5 +1,5 @@ use hare::ast; -use hare::lex::{btoken}; +use hare::lex::{ltok}; use hare::lex; use io; @@ -8,25 +8,3 @@ export fn expression(lexer: *lex::lexer) (ast::expr | error) = { // TODO: if, for, switch, match, etc return binarithm(lexer, void, 0); }; - -// Parses a compound-expression. -export fn compound_expression(lexer: *lex::lexer) (ast::expr | error) = { - let tok = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Unexpected EOF, expected compound expression"), - t: (lex::token, lex::location) => t, - }; - - lex::unlex(lexer, tok); - let tok = match (tok.0) { - tok: btoken => tok, - * => return expression(lexer), - }; - - return switch (tok) { - btoken::LBRACE => expression_list(lexer), - btoken::BREAK, btoken::CONTINUE, btoken::RETURN => - control_statement(lexer), - * => expression(lexer), - }; -}; diff --git a/hare/parse/ident.ha b/hare/parse/ident.ha @@ -1,18 +1,18 @@ use hare::ast; use hare::lex; -use hare::lex::{btoken}; +use hare::lex::{ltok}; fn ident_trailing(lexer: *lex::lexer) ((ast::ident, bool) | error) = { let ident: []str = []; let z = 0z; for (true) { - let name = match (try_name(lexer)?) { - n: lex::name => n, + let name = match (try_tok(lexer, ltok::NAME)?) { + t: lex::token => t.1 as str, void => return (ident: ast::ident, true), }; - append(ident, name: str); + append(ident, name); z += len(name); - match (try_btoken(lexer, btoken::DOUBLE_COLON)?) { + match (try_tok(lexer, ltok::DOUBLE_COLON)?) { void => break, * => void, // Grab the next ident }; diff --git a/hare/parse/import.ha b/hare/parse/import.ha @@ -1,17 +1,17 @@ use hare::ast; use hare::lex; -use hare::lex::{btoken}; +use hare::lex::{ltok}; fn name_list(lexer: *lex::lexer) ([]str | error) = { let names: []str = []; for (true) { - append(names, want_name(lexer)?: str); - switch (want_btoken(lexer, btoken::COMMA, btoken::RBRACE)?) { - btoken::COMMA => match (try_btoken(lexer, btoken::RBRACE)?) { + append(names, want_tok(lexer, ltok::NAME)?.1 as str); + switch (want_tok(lexer, ltok::COMMA, ltok::RBRACE)?.0) { + ltok::COMMA => match (try_tok(lexer, ltok::RBRACE)?) { void => void, * => return names, }, - btoken::RBRACE => return names, + ltok::RBRACE => return names, * => abort(), // Unreachable }; }; @@ -22,21 +22,21 @@ fn name_list(lexer: *lex::lexer) ([]str | error) = { export fn imports(lexer: *lex::lexer) ([]ast::import | error) = { let imports: []ast::import = []; for (true) { - match (try_btoken(lexer, btoken::USE)?) { + match (try_tok(lexer, ltok::USE)?) { void => break, * => void, }; let name = ident_trailing(lexer)?; - switch (want_btoken(lexer, btoken::SEMICOLON, btoken::LBRACE, - btoken::EQUAL)?) { - btoken::SEMICOLON => { + switch (want_tok(lexer, ltok::SEMICOLON, ltok::LBRACE, + ltok::EQUAL)?.0) { + ltok::SEMICOLON => { synassert(mkloc(lexer), !name.1, "Unexpected trailing :: in ident")?; append(imports, name.0: ast::import_module); }, - btoken::LBRACE => { + ltok::LBRACE => { synassert(mkloc(lexer), name.1, "Expected trailing :: in ident")?; let objects = name_list(lexer)?; @@ -44,9 +44,9 @@ export fn imports(lexer: *lex::lexer) ([]ast::import | error) = { ident = name.0, objects = objects, }); - want_btoken(lexer, btoken::SEMICOLON)?; + want_tok(lexer, ltok::SEMICOLON)?; }, - btoken::EQUAL => { + ltok::EQUAL => { synassert(mkloc(lexer), len(name.0) == 1 && !name.1, "Expected name, not ident")?; @@ -55,7 +55,7 @@ export fn imports(lexer: *lex::lexer) ([]ast::import | error) = { ident = ident, alias = name.0[0], }); - want_btoken(lexer, btoken::SEMICOLON)?; + want_tok(lexer, ltok::SEMICOLON)?; }, * => abort(), // Unreachable }; diff --git a/hare/parse/type.ha b/hare/parse/type.ha @@ -1,50 +1,50 @@ use hare::ast; use hare::ast::{builtin_type}; use hare::lex; -use hare::lex::{btoken}; +use hare::lex::{ltok}; fn prototype(lexer: *lex::lexer) (ast::func_type | error) = { let variadism = ast::variadism::NONE; let params: []ast::func_param = []; - want_btoken(lexer, btoken::LPAREN)?; - for (try_btoken(lexer, btoken::RPAREN)? is void) { + want_tok(lexer, ltok::LPAREN)?; + for (try_tok(lexer, ltok::RPAREN)? is void) { let loc = mkloc(lexer); - match (try_btoken(lexer, btoken::ELLIPSIS)?) { + match (try_tok(lexer, ltok::ELLIPSIS)?) { void => void, - lex::btoken => { + lex::token => { synassert(loc, len(params) > 0, "Expected at least one non-variadic parameter for C-style variadism")?; variadism = ast::variadism::C; - try_btoken(lexer, btoken::COMMA)?; - want_btoken(lexer, btoken::RPAREN)?; + try_tok(lexer, ltok::COMMA)?; + want_tok(lexer, ltok::RPAREN)?; break; }, }; - let name = match (try_btoken(lexer, btoken::UNDERSCORE)?) { - void => want_name(lexer)?: str, - lex::btoken => "", + let name = match (try_tok(lexer, ltok::UNDERSCORE)?) { + void => want_tok(lexer, ltok::NAME)?.1 as str, + lex::token => "", }; - want_btoken(lexer, btoken::COLON); + want_tok(lexer, ltok::COLON); append(params, ast::func_param { loc = loc, name = name, _type = alloc(_type(lexer)?), }); - match (try_btoken(lexer, btoken::ELLIPSIS)?) { + match (try_tok(lexer, ltok::ELLIPSIS)?) { void => void, - lex::btoken => { + lex::token => { variadism = ast::variadism::HARE; - try_btoken(lexer, btoken::COMMA)?; - want_btoken(lexer, btoken::RPAREN)?; + try_tok(lexer, ltok::COMMA)?; + want_tok(lexer, ltok::RPAREN)?; break; }, }; - match (try_btoken(lexer, btoken::COMMA)?) { + match (try_tok(lexer, ltok::COMMA)?) { void => { - want_btoken(lexer, btoken::RPAREN)?; + want_tok(lexer, ltok::RPAREN)?; break; }, - lex::btoken => void, + lex::token => void, }; }; let t = _type(lexer)?; @@ -58,53 +58,45 @@ fn prototype(lexer: *lex::lexer) (ast::func_type | error) = { fn integer_type( lexer: *lex::lexer, -) (builtin_type | error) = switch (want_btoken(lexer)?) { - btoken::CHAR => builtin_type::CHAR, - btoken::I16 => builtin_type::I16, - btoken::I32 => builtin_type::I32, - btoken::I64 => builtin_type::I64, - btoken::I64 => builtin_type::I64, - btoken::I8 => builtin_type::I8, - btoken::INT => builtin_type::INT, - btoken::SIZE => builtin_type::SIZE, - btoken::U16 => builtin_type::U16, - btoken::U32 => builtin_type::U32, - btoken::U64 => builtin_type::U64, - btoken::U64 => builtin_type::U64, - btoken::U8 => builtin_type::U8, - btoken::UINT => builtin_type::UINT, - btoken::UINTPTR => builtin_type::UINTPTR, +) (builtin_type | error) = switch (want_tok(lexer)?.0) { + ltok::CHAR => builtin_type::CHAR, + ltok::I16 => builtin_type::I16, + ltok::I32 => builtin_type::I32, + ltok::I64 => builtin_type::I64, + ltok::I64 => builtin_type::I64, + ltok::I8 => builtin_type::I8, + ltok::INT => builtin_type::INT, + ltok::SIZE => builtin_type::SIZE, + ltok::U16 => builtin_type::U16, + ltok::U32 => builtin_type::U32, + ltok::U64 => builtin_type::U64, + ltok::U64 => builtin_type::U64, + ltok::U8 => builtin_type::U8, + ltok::UINT => builtin_type::UINT, + ltok::UINTPTR => builtin_type::UINTPTR, * => syntaxerr(mkloc(lexer), "Expected integer type"), }; fn primitive_type(lexer: *lex::lexer) (ast::_type | error) = { - let t = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Expected primitive type, got EOF"), - t: (lex::token, lex::location) => t, - }; - let b = match (t.0) { - b: lex::btoken => b, - * => return syntaxerr(mkloc(lexer), - "Unexpected {}, was expecting primitive type", - lex::tokstr(t.0)), - }; - let builtin = switch (b) { - btoken::CHAR, btoken::I16, btoken::I32, btoken::I64, - btoken::I64, btoken::I8, btoken::INT, btoken::SIZE, btoken::U16, - btoken::U32, btoken::U64, btoken::U64, btoken::U8, btoken::UINT, - btoken::UINTPTR => { - lex::unlex(lexer, t); + let tok = want_tok(lexer)?; + let builtin = switch (tok.0) { + ltok::CHAR, ltok::I16, ltok::I32, ltok::I64, + ltok::I64, ltok::I8, ltok::INT, ltok::SIZE, ltok::U16, + ltok::U32, ltok::U64, ltok::U64, ltok::U8, ltok::UINT, + ltok::UINTPTR => { + lex::unlex(lexer, tok); integer_type(lexer)?; }, - btoken::RUNE => builtin_type::RUNE, - btoken::STR => builtin_type::STR, - btoken::F32 => builtin_type::F32, - btoken::F64 => builtin_type::F64, - btoken::BOOL => builtin_type::BOOL, - btoken::VOID => builtin_type::VOID, - btoken::NULL => builtin_type::NULL, - * => return syntaxerr(mkloc(lexer), "Expected primitive type"), + ltok::RUNE => builtin_type::RUNE, + ltok::STR => builtin_type::STR, + ltok::F32 => builtin_type::F32, + ltok::F64 => builtin_type::F64, + ltok::BOOL => builtin_type::BOOL, + ltok::VOID => builtin_type::VOID, + ltok::NULL => builtin_type::NULL, + * => return syntaxerr(mkloc(lexer), + "Unexected {}, was expecting primitive type", + lex::tokstr(tok)), }; return ast::_type { loc = mkloc(lexer), @@ -115,7 +107,7 @@ fn primitive_type(lexer: *lex::lexer) (ast::_type | error) = { fn alias_type(lexer: *lex::lexer) (ast::_type | error) = { let loc = mkloc(lexer); - let unwrap = match (try_btoken(lexer, btoken::ELLIPSIS)?) { + let unwrap = match (try_tok(lexer, ltok::ELLIPSIS)?) { void => false, * => true, }; @@ -131,11 +123,11 @@ fn alias_type(lexer: *lex::lexer) (ast::_type | error) = { fn pointer_type(lexer: *lex::lexer) (ast::_type | error) = { let loc = mkloc(lexer); - let flags = match (try_btoken(lexer, btoken::NULLABLE)?) { + let flags = match (try_tok(lexer, ltok::NULLABLE)?) { void => 0: ast::pointer_flags, * => ast::pointer_flags::NULLABLE, }; - want_btoken(lexer, btoken::TIMES)?; + want_tok(lexer, ltok::TIMES)?; return ast::_type { loc = loc, flags = 0, @@ -150,14 +142,14 @@ fn tagged_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = { let loc = mkloc(lexer); let tagged: ast::tagged_type = []; append(tagged, alloc(first)); - for (try_btoken(lexer, btoken::RPAREN)? is void) { + for (try_tok(lexer, ltok::RPAREN)? is void) { append(tagged, alloc(_type(lexer)?)); - match (try_btoken(lexer, btoken::BOR)?) { + match (try_tok(lexer, ltok::BOR)?) { void => { - want_btoken(lexer, btoken::RPAREN)?; + want_tok(lexer, ltok::RPAREN)?; break; }, - lex::btoken => void, + lex::token => void, }; }; return ast::_type { @@ -171,14 +163,14 @@ fn tuple_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = { let loc = mkloc(lexer); let tuple: ast::tuple_type = []; append(tuple, alloc(first)); - for (try_btoken(lexer, btoken::RPAREN)? is void) { + for (try_tok(lexer, ltok::RPAREN)? is void) { append(tuple, alloc(_type(lexer)?)); - match (try_btoken(lexer, btoken::COMMA)?) { + match (try_tok(lexer, ltok::COMMA)?) { void => { - want_btoken(lexer, btoken::RPAREN)?; + want_tok(lexer, ltok::RPAREN)?; break; }, - lex::btoken => void, + lex::token => void, }; }; return ast::_type { @@ -190,11 +182,11 @@ fn tuple_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = { fn fn_type(lexer: *lex::lexer) (ast::_type | error) = { let loc = mkloc(lexer); - let attrs = match (try_btoken(lexer, btoken::ATTR_NORETURN)?) { + let attrs = match (try_tok(lexer, ltok::ATTR_NORETURN)?) { void => 0: ast::func_attrs, * => ast::func_attrs::NORETURN, }; - want_btoken(lexer, btoken::FN)?; + want_tok(lexer, ltok::FN)?; let proto = prototype(lexer)?; proto.attrs |= attrs; return ast::_type { @@ -206,49 +198,40 @@ fn fn_type(lexer: *lex::lexer) (ast::_type | error) = { // Parses a type export fn _type(lexer: *lex::lexer) (ast::_type | error) = { - let flags: ast::type_flags = match (try_btoken(lexer, btoken::CONST)?) { + let flags: ast::type_flags = match (try_tok(lexer, ltok::CONST)?) { void => 0, * => ast::type_flags::CONST, }; - let t = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Unexpected EOF, was expecting type"), - t: (lex::token, lex::location) => t, - }; - lex::unlex(lexer, t); - let typ: ast::_type = match (t.0) { - b: lex::btoken => switch (b) { - btoken::CHAR, btoken::I16, btoken::I32, btoken::I64, - btoken::I64, btoken::I8, btoken::INT, btoken::SIZE, - btoken::U16, btoken::U32, btoken::U64, btoken::U64, - btoken::U8, btoken::UINT, btoken::UINTPTR, btoken::RUNE, - btoken::STR, btoken::F32, btoken::F64, btoken::BOOL, - btoken::VOID, btoken::NULL => primitive_type(lexer)?, - btoken::ENUM => abort(), // TODO - btoken::NULLABLE, btoken::TIMES => pointer_type(lexer)?, - btoken::STRUCT, btoken::UNION => abort(), // TODO - btoken::LBRACKET => abort(), // TODO - btoken::LPAREN => { - want_btoken(lexer, btoken::LPAREN)?; - let t = _type(lexer)?; - switch (want_btoken(lexer, btoken::BOR, - btoken::COMMA)?) { - btoken::BOR => tagged_type(lexer, t)?, - btoken::COMMA => tuple_type(lexer, t)?, - * => abort("unreachable"), - }; - }, - btoken::ATTR_NORETURN, btoken::FN => fn_type(lexer)?, - btoken::ELLIPSIS => alias_type(lexer)?, - * => return syntaxerr(mkloc(lexer), "Expected type"), + let tok = peek_tok(lexer)? as lex::token; + let typ: ast::_type = switch (tok.0) { + ltok::CHAR, ltok::I16, ltok::I32, ltok::I64, + ltok::I64, ltok::I8, ltok::INT, ltok::SIZE, + ltok::U16, ltok::U32, ltok::U64, ltok::U64, + ltok::U8, ltok::UINT, ltok::UINTPTR, ltok::RUNE, + ltok::STR, ltok::F32, ltok::F64, ltok::BOOL, + ltok::VOID, ltok::NULL => primitive_type(lexer)?, + ltok::ENUM => abort(), // TODO + ltok::NULLABLE, ltok::TIMES => pointer_type(lexer)?, + ltok::STRUCT, ltok::UNION => abort(), // TODO + ltok::LBRACKET => abort(), // TODO + ltok::LPAREN => { + want_tok(lexer, ltok::LPAREN)?; + let t = _type(lexer)?; + switch (want_tok(lexer, ltok::BOR, + ltok::COMMA)?.0) { + ltok::BOR => tagged_type(lexer, t)?, + ltok::COMMA => tuple_type(lexer, t)?, + * => abort("unreachable"), + }; }, - lex::name => alias_type(lexer)?, + ltok::ATTR_NORETURN, ltok::FN => fn_type(lexer)?, + ltok::ELLIPSIS, ltok::NAME => alias_type(lexer)?, * => return syntaxerr(mkloc(lexer), "Unexpected {}, was expecting type", - lex::tokstr(t.0)), + lex::tokstr(tok)), }; - match (try_btoken(lexer, btoken::LNOT)?) { + match (try_tok(lexer, ltok::LNOT)?) { void => void, * => flags |= ast::type_flags::ERROR, }; diff --git a/hare/parse/util.ha b/hare/parse/util.ha @@ -4,61 +4,23 @@ use hare::lex; use io; use strio; -// Requires the next token to be a name. Returns that name, or an error. -fn want_name(lexer: *lex::lexer) (lex::name | error) = { - match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), - "Unexpected EOF, was expecting name"), - t: (lex::token, lex::location) => match (t.0) { - n: lex::name => return n, - * => return syntaxerr(mkloc(lexer), - "Unexpected {}, was expecting name", - lex::tokstr(t.0)), - }, +// Requires the next token to have a matching ltok. Returns that token, or an +// error. +fn want_tok(lexer: *lex::lexer, want: lex::ltok...) (lex::token | error) = { + let tok = lex::lex(lexer)?; + if (len(want) == 0) { + return tok; }; -}; - -// Looks for a matching name from the lexer, and if not present, unlexes the -// token and returns void. If found, the token is consumed from the lexer and is -// returned. -fn try_name(lexer: *lex::lexer) (lex::name | error | void) = { - let tuple = match (lex::lex(lexer)?) { - io::EOF => return, - t: (lex::token, lex::location) => match (t.0) { - n: lex::name => return n, - * => t, - }, - }; - lex::unlex(lexer, tuple); -}; - -// Requires the next token to be a name. Returns that name, or an error. -fn want_btoken( - lexer: *lex::lexer, - want: lex::btoken... -) (lex::btoken | error) = { - let tok: lex::token = match (lex::lex(lexer)?) { - io::EOF => return syntaxerr(mkloc(lexer), "Unexpected EOF"), - t: (lex::token, lex::location) => match (t.0) { - b: lex::btoken => { - if (len(want) == 0) { - return b; - }; - for (let i = 0z; i < len(want); i += 1) { - if (b == want[i]) { - return b; - }; - }; - t.0; - }, - * => t.0, - }, + for (let i = 0z; i < len(want); i += 1) { + if (tok.0 == want[i]) { + return tok; + }; }; let buf = strio::dynamic(); defer io::close(buf); for (let i = 0z; i < len(want); i += 1) { - fmt::fprintf(buf, lex::tokstr(want[i])); + fmt::fprintf(buf, lex::tokstr((want[i], void, mkloc(lexer)))); if (i + 1 < len(want)) { fmt::fprint(buf, ", "); }; @@ -67,64 +29,38 @@ fn want_btoken( lex::tokstr(tok), strio::string(buf)); }; -// Looks for a matching btoken from the lexer, and if not present, unlexes the +// Looks for a matching ltok from the lexer, and if not present, unlexes the // token and returns void. If found, the token is consumed from the lexer and is // returned. -fn try_btoken( +fn try_tok( lexer: *lex::lexer, - want: lex::btoken... -) (lex::btoken | error | void) = { - let tok = lex::lex(lexer); - let tuple = match (tok?) { - io::EOF => return, - t: (lex::token, lex::location) => { - match (t.0) { - b: lex::btoken => - for (let i = 0z; i < len(want); i += 1) { - if (b == want[i]) { - return b; - }; - }, - * => void, - }; - t; - }, + want: lex::ltok... +) (lex::token | error | void) = { + let tok = lex::lex(lexer)?; + assert(len(want) > 0); + for (let i = 0z; i < len(want); i += 1) { + if (tok.0 == want[i]) { + return tok; + }; }; - lex::unlex(lexer, tuple); + lex::unlex(lexer, tok); }; -// Looks for a matching btoken from the lexer, unlexes the token, and returns -// it; or void if it was not a btoken. -fn peek_btoken( +// Looks for a matching ltok from the lexer, unlexes the token, and returns +// it; or void if it was not a ltok. +fn peek_tok( lexer: *lex::lexer, - want: lex::btoken... -) (lex::btoken | error | void) = { - let tok = lex::lex(lexer); - let tuple = match (tok?) { - io::EOF => return, - t: (lex::token, lex::location) => { - match (t.0) { - b: lex::btoken => - for (let i = 0z; i < len(want); i += 1) { - if (b == want[i]) { - lex::unlex(lexer, t); - return b; - }; - }, - * => void, - }; - t; - }, + want: lex::ltok... +) (lex::token | error | void) = { + let tok = lex::lex(lexer)?; + lex::unlex(lexer, tok); + if (len(want) == 0) { + return tok; }; - lex::unlex(lexer, tuple); -}; - -// Identical to [lex::lex], but considers io::EOF a syntax error. -fn mustlex(lexer: *lex::lexer) ((lex::token, lex::location) | error) = { - return match (lex::lex(lexer)) { - err: lex::error => err, - io::EOF => syntaxerr(mkloc(lexer), "Unexpected EOF"), - t: (lex::token, lex::location) => t, + for (let i = 0z; i < len(want); i += 1) { + if (tok.0 == want[i]) { + return tok; + }; }; };