commit bf6029d3dc85a27873063a105c19265b599908ce
parent ecf1617e897fd290a5a8ea5e31c870c39c779faf
Author: Eyal Sawady <ecs@d2evs.net>
Date: Sun, 11 Apr 2021 12:18:35 -0400
Refactor hare::lex and hare::parse
This does a lot of things which all necessarily had to happen at once:
- Move lex::location into lex::token
- Move lex::name and lex::literal into lex::ltok, and add lex::value for
storing the value associated with them
- Replace io::EOF with ltok::EOF
- Drop want_name et al, and return lex::token rather than ltok from
want_tok et al
- Use want_tok et al more pervasively now that they return lex::token
Diffstat:
12 files changed, 572 insertions(+), 777 deletions(-)
diff --git a/hare/ast/expr.ha b/hare/ast/expr.ha
@@ -138,7 +138,7 @@ export type struct_constant = struct {
export type tuple_constant = []*expr;
// A constant
-export type constant_expr = (void | lex::literal | array_constant |
+export type constant_expr = (void | lex::value | array_constant |
struct_constant | tuple_constant);
// continue :label
@@ -318,7 +318,7 @@ export fn expr_free(e: (expr | nullable *expr)) void = match (e) {
expr_free(c._type);
},
c: constant_expr => match(c) {
- (void | lex::literal) => void,
+ (void | lex::value) => void,
a: array_constant => {
for (let i = 0z; i < len(a.values); i += 1) {
expr_free(a.values[i]);
diff --git a/hare/lex/+test.ha b/hare/lex/+test.ha
@@ -19,19 +19,19 @@ use strings;
@test fn unlex() void = {
let lexer = init(io::empty, "<test>");
- unlex(&lexer, (btoken::IF, location {
+ unlex(&lexer, (ltok::IF, void, location {
path = "<test>",
line = 1234,
col = 1234,
}));
- let t = lex(&lexer) as (token, location);
- assert(t.0 is btoken);
- assert(t.0 as btoken == btoken::IF);
- assert(t.1.path == "<test>");
- assert(t.1.line == 1234 && t.1.col == 1234);
+ let t = lex(&lexer) as token;
+ assert(t.0 == ltok::IF);
+ assert(t.1 is void);
+ assert(t.2.path == "<test>");
+ assert(t.2.line == 1234 && t.2.col == 1234);
};
-fn litassert(expected: literal, actual: literal) void = match (expected) {
+fn vassert(expected: value, actual: value) void = match (expected) {
e: u8 => assert(actual as u8 == e),
e: u16 => assert(actual as u16 == e),
e: u32 => assert(actual as u32 == e),
@@ -49,68 +49,48 @@ fn litassert(expected: literal, actual: literal) void = match (expected) {
e: fconst => assert(actual as fconst == e),
e: rune => assert(actual as rune == e),
e: str => assert(actual as str == e),
+ e: void => assert(actual is void),
};
-fn lextest(in: str, expected: [](uint, uint, token)) void = {
+fn lextest(in: str, expected: []token) void = {
let buf = bufio::fixed(strings::toutf8(in), mode::READ);
let lexer = init(buf, "<test>");
for (let i = 0z; i < len(expected); i += 1) {
- let eline = expected[i].0, ecol = expected[i].1,
- etok = expected[i].2;
+ let etok = expected[i];
let tl = match (lex(&lexer)) {
- tl: (token, location) => tl,
- io::EOF => {
- fmt::errorfln("unexpected EOF at {}", i);
- abort();
- },
+ tl: token => tl,
err: error => {
fmt::errorfln("{}: {}", i, strerror(err));
abort();
},
};
- let tok = tl.0, loc = tl.1;
- match (tok) {
- b: btoken => if (!(etok is btoken) || etok as btoken != b) {
- fmt::errorfln("bad token at {}: got {}, wanted {}",
- i, tokstr(tok), tokstr(etok));
- abort();
- },
- n: name => if (!(etok is name) || etok as name != n) {
- fmt::errorfln("bad token at {}: got {}, wanted {}",
- i, tokstr(tok), tokstr(etok));
- abort();
- },
- l: literal => if (!(etok is literal)) {
- fmt::errorfln("bad token at {}: got {}, wanted {}",
- i, tokstr(tok), tokstr(etok));
- abort();
- } else {
- litassert(l, etok as literal);
- },
- * => abort("TODO"),
- };
- assert(loc.path == "<test>");
- if (loc.line != eline || loc.col != ecol) {
- fmt::errorfln("bad line/col at {}: got {},{}; wanted {},{}",
- i, loc.line, loc.col, eline, ecol);
- abort();
- };
+ assert(tl.0 == etok.0);
+ vassert(tl.1, etok.1);
+ assert(tl.2.line == etok.2.line && tl.2.col == etok.2.col
+ && tl.2.path == etok.2.path);
};
- assert(lex(&lexer) is io::EOF);
+ let t = lex(&lexer) as token;
+ assert(t.0 == ltok::EOF);
+};
+
+fn loc(line: uint, col: uint) location = location {
+ path = "<test>",
+ line = line,
+ col = col,
};
@test fn lex1() void = {
const in = "~,{[(}]);";
- const expected: [_](uint, uint, token) = [
- (1, 1, btoken::BNOT),
- (1, 2, btoken::COMMA),
- (1, 3, btoken::LBRACE),
- (1, 4, btoken::LBRACKET),
- (1, 5, btoken::LPAREN),
- (1, 6, btoken::RBRACE),
- (1, 7, btoken::RBRACKET),
- (1, 8, btoken::RPAREN),
- (1, 9, btoken::SEMICOLON),
+ const expected: [_]token = [
+ (ltok::BNOT, void, loc(1, 1)),
+ (ltok::COMMA, void, loc(1, 2)),
+ (ltok::LBRACE, void, loc(1, 3)),
+ (ltok::LBRACKET, void, loc(1, 4)),
+ (ltok::LPAREN, void, loc(1, 5)),
+ (ltok::RBRACE, void, loc(1, 6)),
+ (ltok::RBRACKET, void, loc(1, 7)),
+ (ltok::RPAREN, void, loc(1, 8)),
+ (ltok::SEMICOLON, void, loc(1, 9)),
];
lextest(in, expected);
};
@@ -118,88 +98,83 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = {
@test fn lex2() void = {
// Ends with = to test =, EOF
const in = "^ ^^ ^= * *= % %= + += - -= : :: & && &= | || |= = == / /= =";
- const expected: [_](uint, uint, token) = [
- (1, 1, btoken::BXOR),
- (1, 3, btoken::LXOR),
- (1, 6, btoken::BXOREQ),
- (1, 9, btoken::TIMES),
- (1, 11, btoken::TIMESEQ),
- (1, 14, btoken::MODULO),
- (1, 16, btoken::MODEQ),
- (1, 19, btoken::PLUS),
- (1, 21, btoken::PLUSEQ),
- (1, 24, btoken::MINUS),
- (1, 26, btoken::MINUSEQ),
- (1, 29, btoken::COLON),
- (1, 31, btoken::DOUBLE_COLON),
- (1, 34, btoken::BAND),
- (1, 36, btoken::LAND),
- (1, 39, btoken::ANDEQ),
- (1, 42, btoken::BOR),
- (1, 44, btoken::LOR),
- (1, 47, btoken::OREQ),
- (1, 50, btoken::EQUAL),
- (1, 52, btoken::LEQUAL),
- (1, 55, btoken::DIV),
- (1, 57, btoken::DIVEQ),
- (1, 60, btoken::EQUAL),
+ const expected: [_]token = [
+ (ltok::BXOR, void, loc(1, 1)),
+ (ltok::LXOR, void, loc(1, 3)),
+ (ltok::BXOREQ, void, loc(1, 6)),
+ (ltok::TIMES, void, loc(1, 9)),
+ (ltok::TIMESEQ, void, loc(1, 11)),
+ (ltok::MODULO, void, loc(1, 14)),
+ (ltok::MODEQ, void, loc(1, 16)),
+ (ltok::PLUS, void, loc(1, 19)),
+ (ltok::PLUSEQ, void, loc(1, 21)),
+ (ltok::MINUS, void, loc(1, 24)),
+ (ltok::MINUSEQ, void, loc(1, 26)),
+ (ltok::COLON, void, loc(1, 29)),
+ (ltok::DOUBLE_COLON, void, loc(1, 31)),
+ (ltok::BAND, void, loc(1, 34)),
+ (ltok::LAND, void, loc(1, 36)),
+ (ltok::ANDEQ, void, loc(1, 39)),
+ (ltok::BOR, void, loc(1, 42)),
+ (ltok::LOR, void, loc(1, 44)),
+ (ltok::OREQ, void, loc(1, 47)),
+ (ltok::EQUAL, void, loc(1, 50)),
+ (ltok::LEQUAL, void, loc(1, 52)),
+ (ltok::DIV, void, loc(1, 55)),
+ (ltok::DIVEQ, void, loc(1, 57)),
+ (ltok::EQUAL, void, loc(1, 60)),
];
lextest(in, expected);
};
@test fn lex3() void = {
const in = ". .. ... < << <= <<= > >> >= >>= >>";
- const expected: [_](uint, uint, token) = [
- (1, 1, btoken::DOT),
- (1, 3, btoken::SLICE),
- (1, 6, btoken::ELLIPSIS),
- (1, 10, btoken::LESS),
- (1, 12, btoken::LSHIFT),
- (1, 15, btoken::LESSEQ),
- (1, 18, btoken::LSHIFTEQ),
- (1, 22, btoken::GREATER),
- (1, 24, btoken::RSHIFT),
- (1, 27, btoken::GREATEREQ),
- (1, 30, btoken::RSHIFTEQ),
- (1, 34, btoken::RSHIFT),
+ const expected: [_]token = [
+ (ltok::DOT, void, loc(1, 1)),
+ (ltok::SLICE, void, loc(1, 3)),
+ (ltok::ELLIPSIS, void, loc(1, 6)),
+ (ltok::LESS, void, loc(1, 10)),
+ (ltok::LSHIFT, void, loc(1, 12)),
+ (ltok::LESSEQ, void, loc(1, 15)),
+ (ltok::LSHIFTEQ, void, loc(1, 18)),
+ (ltok::GREATER, void, loc(1, 22)),
+ (ltok::RSHIFT, void, loc(1, 24)),
+ (ltok::GREATEREQ, void, loc(1, 27)),
+ (ltok::RSHIFTEQ, void, loc(1, 30)),
+ (ltok::RSHIFT, void, loc(1, 34)),
];
lextest(in, expected);
};
@test fn lexname() void = {
const in = "hello world return void foobar";
- const expected: [_](uint, uint, token) = [
- (1, 1, "hello": name),
- (1, 7, "world": name),
- (1, 13, btoken::RETURN),
- (1, 20, btoken::VOID),
- (1, 25, "foobar": name),
+ const expected: [_]token = [
+ (ltok::NAME, "hello", loc(1, 1)),
+ (ltok::NAME, "world", loc(1, 7)),
+ (ltok::RETURN, void, loc(1, 13)),
+ (ltok::VOID, void, loc(1, 20)),
+ (ltok::NAME, "foobar", loc(1, 25)),
];
lextest(in, expected);
};
@test fn keywords() void = {
- let keywords = bmap[..btoken::LAST_KEYWORD+1];
+ let keywords = bmap[..ltok::LAST_KEYWORD+1];
for (let i = 0z; i < len(keywords); i += 1) {
let lexer = init(bufio::fixed(
strings::toutf8(keywords[i]), mode::READ),
"<test>");
- let tl = match (lex(&lexer)) {
- tl: (token, location) => tl,
- * => abort(),
- };
- let tok = tl.0;
- assert(tok is btoken);
- assert(tok as btoken == i: btoken);
+ let tok = lex(&lexer) as token;
+ assert(tok.0 == i: ltok);
};
};
@test fn comments() void = {
const in = "hello world // foo\nbar";
- const expected: [_](uint, uint, token) = [
- (1, 1, "hello": name),
- (1, 7, "world": name),
- (2, 1, "bar": name),
+ const expected: [_]token = [
+ (ltok::NAME, "hello", loc(1, 1)),
+ (ltok::NAME, "world", loc(1, 7)),
+ (ltok::NAME, "bar", loc(2, 1)),
];
lextest(in, expected);
};
@@ -207,22 +182,22 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = {
@test fn runes() void = {
const in = "'a' 'b' '\\a' '\\b' '\\f' '\\n' '\\r' '\\t' '\\v' '\\0' "
"'\\\\' '\\\'' '\\x0A' '\\u1234' '\\U12345678'";
- const expected: [_](uint, uint, token) = [
- (1, 1, 'a'),
- (1, 5, 'b'),
- (1, 9, '\a'),
- (1, 14, '\b'),
- (1, 19, '\f'),
- (1, 24, '\n'),
- (1, 29, '\r'),
- (1, 34, '\t'),
- (1, 39, '\v'),
- (1, 44, '\0'),
- (1, 49, '\\'),
- (1, 54, '\''),
- (1, 59, '\x0A'),
- (1, 66, '\u1234'),
- (1, 75, '\U12345678'),
+ const expected: [_]token = [
+ (ltok::LITERAL, 'a', loc(1, 1)),
+ (ltok::LITERAL, 'b', loc(1, 5)),
+ (ltok::LITERAL, '\a', loc(1, 9)),
+ (ltok::LITERAL, '\b', loc(1, 14)),
+ (ltok::LITERAL, '\f', loc(1, 19)),
+ (ltok::LITERAL, '\n', loc(1, 24)),
+ (ltok::LITERAL, '\r', loc(1, 29)),
+ (ltok::LITERAL, '\t', loc(1, 34)),
+ (ltok::LITERAL, '\v', loc(1, 39)),
+ (ltok::LITERAL, '\0', loc(1, 44)),
+ (ltok::LITERAL, '\\', loc(1, 49)),
+ (ltok::LITERAL, '\'', loc(1, 54)),
+ (ltok::LITERAL, '\x0A', loc(1, 59)),
+ (ltok::LITERAL, '\u1234', loc(1, 66)),
+ (ltok::LITERAL, '\U12345678', loc(1, 75)),
];
lextest(in, expected);
};
@@ -230,33 +205,33 @@ fn lextest(in: str, expected: [](uint, uint, token)) void = {
@test fn strings() void = {
const in = "\"a\" \"b\" \"\\a\" \"\\b\" \"\\f\" \"\\n\" \"\\r\" "
"\"\\t\" \"\\v\" \"\\0\" \"\\\\\" \"\\\'\"";
- const expected: [_](uint, uint, token) = [
- (1, 1, "a": literal),
- (1, 5, "b": literal),
- (1, 9, "\a": literal),
- (1, 14, "\b": literal),
- (1, 19, "\f": literal),
- (1, 24, "\n": literal),
- (1, 29, "\r": literal),
- (1, 34, "\t": literal),
- (1, 39, "\v": literal),
- (1, 44, "\0": literal),
- (1, 49, "\\": literal),
- (1, 54, "\'": literal),
+ const expected: [_]token = [
+ (ltok::LITERAL, "a", loc(1, 1)),
+ (ltok::LITERAL, "b", loc(1, 5)),
+ (ltok::LITERAL, "\a", loc(1, 9)),
+ (ltok::LITERAL, "\b", loc(1, 14)),
+ (ltok::LITERAL, "\f", loc(1, 19)),
+ (ltok::LITERAL, "\n", loc(1, 24)),
+ (ltok::LITERAL, "\r", loc(1, 29)),
+ (ltok::LITERAL, "\t", loc(1, 34)),
+ (ltok::LITERAL, "\v", loc(1, 39)),
+ (ltok::LITERAL, "\0", loc(1, 44)),
+ (ltok::LITERAL, "\\", loc(1, 49)),
+ (ltok::LITERAL, "\'", loc(1, 54)),
];
// TODO: test \x and \u and \U
lextest(in, expected);
const in = "\"ab\\a\\b\\f\\n\\r\\t\\v\\0\\\\\\'\"";
- const expected: [_](uint, uint, token) = [
- (1, 1, "ab\a\b\f\n\r\t\v\0\\\'": literal),
+ const expected: [_]token = [
+ (ltok::LITERAL, "ab\a\b\f\n\r\t\v\0\\\'", loc(1, 1)),
];
lextest(in, expected);
const in = "\"hello world\" \"こんにちは\" \"return\" \"foo\"";
- const expected: [_](uint, uint, token) = [
- (1, 1, "hello world": literal),
- (1, 15, "こんにちは": literal),
- (1, 23, "return": literal),
- (1, 32, "foo": literal),
+ const expected: [_]token = [
+ (ltok::LITERAL, "hello world", loc(1, 1)),
+ (ltok::LITERAL, "こんにちは", loc(1, 15)),
+ (ltok::LITERAL, "return", loc(1, 23)),
+ (ltok::LITERAL, "foo", loc(1, 32)),
];
lextest(in, expected);
};
diff --git a/hare/lex/lex.ha b/hare/lex/lex.ha
@@ -13,7 +13,7 @@ export type lexer = struct {
in: *io::stream,
path: str,
loc: (uint, uint),
- un: ((token, location) | void),
+ un: (token | void),
rb: [2](rune | io::EOF | void),
};
@@ -43,9 +43,9 @@ export fn init(in: *io::stream, path: str) lexer = lexer {
};
// Returns the next token from the lexer.
-export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = {
+export fn lex(lex: *lexer) (token | error) = {
match (lex.un) {
- tok: (token, location) => {
+ tok: token => {
lex.un = void;
return tok;
},
@@ -54,7 +54,7 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = {
let loc = location { ... };
let r: rune = match (nextw(lex)?) {
- io::EOF => return io::EOF,
+ io::EOF => return (ltok::EOF, void, mkloc(lex)),
r: (rune, location) => {
loc = r.1;
r.0;
@@ -70,7 +70,7 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = {
abort(); // TODO: Literals
};
- let tok: token = switch (r) {
+ let tok: ltok = switch (r) {
* => return syntaxerr(loc, "invalid character"),
'"', '\'' => {
unget(lex, r);
@@ -80,18 +80,18 @@ export fn lex(lex: *lexer) ((token, location) | io::EOF | error) = {
'^', '*', '%', '/', '+', '-', ':', '!', '&', '|', '=' => {
return lex2(lex, loc, r);
},
- '~' => btoken::BNOT,
- ',' => btoken::COMMA,
- '{' => btoken::LBRACE,
- '[' => btoken::LBRACKET,
- '(' => btoken::LPAREN,
- '}' => btoken::RBRACE,
- ']' => btoken::RBRACKET,
- ')' => btoken::RPAREN,
- ';' => btoken::SEMICOLON,
- '?' => btoken::QUESTION,
+ '~' => ltok::BNOT,
+ ',' => ltok::COMMA,
+ '{' => ltok::LBRACE,
+ '[' => ltok::LBRACKET,
+ '(' => ltok::LPAREN,
+ '}' => ltok::RBRACE,
+ ']' => ltok::RBRACKET,
+ ')' => ltok::RPAREN,
+ ';' => ltok::SEMICOLON,
+ '?' => ltok::QUESTION,
};
- return (tok, loc);
+ return (tok, void, loc);
};
fn is_name(r: rune, num: bool) bool =
@@ -156,10 +156,7 @@ fn lex_rune(lex: *lexer, loc: location) (rune | error) = {
};
};
-fn lex_string(
- lex: *lexer,
- loc: location,
-) ((token, location) | io::EOF | error) = {
+fn lex_string(lex: *lexer, loc: location) (token | error) = {
let chars: []u8 = [];
for (true) match (next(lex)?) {
io::EOF => return syntaxerr(loc, "unexpected EOF scanning string literal"),
@@ -171,13 +168,10 @@ fn lex_string(
append(chars, utf8::encoderune(r)...);
},
};
- return (strings::fromutf8(chars): literal, loc);
+ return (ltok::LITERAL, strings::fromutf8(chars), loc);
};
-fn lex_rn_str(
- lex: *lexer,
- loc: location,
-) ((token, location) | io::EOF | error) = {
+fn lex_rn_str(lex: *lexer, loc: location) (token | error) = {
let r = match (next(lex)) {
r: rune => r,
(io::EOF | io::error) => abort(),
@@ -189,7 +183,7 @@ fn lex_rn_str(
};
// Rune literal
- let ret: (token, location) = (lex_rune(lex, loc)?: literal, loc);
+ let ret: token = (ltok::LITERAL, lex_rune(lex, loc)?, loc);
match (next(lex)?) {
io::EOF =>
return syntaxerr(loc, "unexpected EOF"),
@@ -199,10 +193,7 @@ fn lex_rn_str(
return ret;
};
-fn lex_name(
- lex: *lexer,
- loc: location,
-) ((token, location) | io::EOF | error) = {
+fn lex_name(lex: *lexer, loc: location) (token | error) = {
let chars: []u8 = [];
match (next(lex)) {
r: rune => {
@@ -224,46 +215,39 @@ fn lex_name(
};
let n = strings::fromutf8(chars);
- return match (sort::search(bmap[..btoken::LAST_KEYWORD+1],
+ return match (sort::search(bmap[..ltok::LAST_KEYWORD+1],
size(str), &n, &ncmp)) {
// TODO: Validate that names are ASCII
- null => (n: name: token, loc),
+ null => (ltok::NAME, n, loc),
v: *void => {
let tok = v: uintptr - &bmap[0]: uintptr;
tok /= size(str): uintptr;
- (tok: btoken: token, loc);
+ (tok: ltok, void, loc);
},
};
};
-fn lex2(
- lexr: *lexer,
- loc: location,
- r: rune,
-) ((token, location) | io::EOF | error) = {
- let n = match (next(lexr)?) {
- io::EOF => io::EOF,
- r: rune => r,
- };
- let tok: token = switch (r) {
+fn lex2(lexr: *lexer, loc: location, r: rune) (token | error) = {
+ let n = next(lexr)?;
+ let tok: ltok = switch (r) {
'^' => match (n) {
r: rune => switch (r) {
- '^' => return (btoken::LXOR: token, loc),
- '=' => return (btoken::BXOREQ: token, loc),
- * => btoken::BXOR,
+ '^' => return (ltok::LXOR, void, loc),
+ '=' => return (ltok::BXOREQ, void, loc),
+ * => ltok::BXOR,
},
- io::EOF => btoken::BXOR,
+ io::EOF => ltok::BXOR,
},
'*' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::TIMESEQ: token, loc),
- * => btoken::TIMES,
+ '=' => return (ltok::TIMESEQ, void, loc),
+ * => ltok::TIMES,
},
- io::EOF => btoken::TIMES,
+ io::EOF => ltok::TIMES,
},
'/' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::DIVEQ: token, loc),
+ '=' => return (ltok::DIVEQ, void, loc),
'/' => {
// Comment
for (true) match (next(lexr)?) {
@@ -274,84 +258,81 @@ fn lex2(
};
return lex(lexr);
},
- * => btoken::DIV,
+ * => ltok::DIV,
},
- io::EOF => btoken::DIV,
+ io::EOF => ltok::DIV,
},
'%' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::MODEQ: token, loc),
- * => btoken::MODULO,
+ '=' => return (ltok::MODEQ, void, loc),
+ * => ltok::MODULO,
},
- io::EOF => btoken::MODULO,
+ io::EOF => ltok::MODULO,
},
'+' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::PLUSEQ: token, loc),
- * => btoken::PLUS,
+ '=' => return (ltok::PLUSEQ, void, loc),
+ * => ltok::PLUS,
},
- io::EOF => btoken::PLUS,
+ io::EOF => ltok::PLUS,
},
'-' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::MINUSEQ: token, loc),
- * => btoken::MINUS,
+ '=' => return (ltok::MINUSEQ, void, loc),
+ * => ltok::MINUS,
},
- io::EOF => btoken::MINUS,
+ io::EOF => ltok::MINUS,
},
':' => match (n) {
r: rune => switch (r) {
- ':' => return (btoken::DOUBLE_COLON: token, loc),
- * => btoken::COLON,
+ ':' => return (ltok::DOUBLE_COLON, void, loc),
+ * => ltok::COLON,
},
- io::EOF => btoken::COLON,
+ io::EOF => ltok::COLON,
},
'!' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::NEQUAL: token, loc),
- * => btoken::LNOT,
+ '=' => return (ltok::NEQUAL, void, loc),
+ * => ltok::LNOT,
},
- io::EOF => btoken::LNOT,
+ io::EOF => ltok::LNOT,
},
'&' => match (n) {
r: rune => switch (r) {
- '&' => return (btoken::LAND: token, loc),
- '=' => return (btoken::ANDEQ: token, loc),
- * => btoken::BAND,
+ '&' => return (ltok::LAND, void, loc),
+ '=' => return (ltok::ANDEQ, void, loc),
+ * => ltok::BAND,
},
- io::EOF => btoken::BAND,
+ io::EOF => ltok::BAND,
},
'|' => match (n) {
r: rune => switch (r) {
- '|' => return (btoken::LOR: token, loc),
- '=' => return (btoken::OREQ: token, loc),
- * => btoken::BOR,
+ '|' => return (ltok::LOR, void, loc),
+ '=' => return (ltok::OREQ, void, loc),
+ * => ltok::BOR,
},
- io::EOF => btoken::BOR,
+ io::EOF => ltok::BOR,
},
'=' => match (n) {
r: rune => switch (r) {
- '=' => return (btoken::LEQUAL: token, loc),
- * => btoken::EQUAL,
+ '=' => return (ltok::LEQUAL, void, loc),
+ * => ltok::EQUAL,
},
- io::EOF => btoken::EQUAL,
+ io::EOF => ltok::EQUAL,
},
* => return syntaxerr(loc, "unknown token sequence"),
};
unget(lexr, n);
- return (tok, loc);
+ return (tok, void, loc);
};
-fn lex3(
- lex: *lexer,
- loc: location,
- r: rune,
-) ((token, location) | io::EOF | error) = {
+fn lex3(lex: *lexer, loc: location, r: rune) (token | error) = {
let n = match (next(lex)?) {
io::EOF => return switch (r) {
- '.' => (btoken::DOT: token, loc),
- '<' => (btoken::LESS: token, loc),
- '>' => (btoken::GREATER: token, loc),
+ '.' => (ltok::DOT, void, loc),
+ '<' => (ltok::LESS, void, loc),
+ '>' => (ltok::GREATER, void, loc),
+ * => abort(), // Invariant
},
r: rune => r,
};
@@ -363,12 +344,8 @@ fn lex3(
};
};
-fn lex3dot(
- lex: *lexer,
- loc: location,
- n: rune,
-) ((token, location) | io::EOF | error) = {
- let tok: token = switch (n) {
+fn lex3dot(lex: *lexer, loc: location, n: rune) (token | error) = {
+ let tok: ltok = switch (n) {
'.' => {
let q = match (next(lex)?) {
io::EOF => io::EOF,
@@ -376,28 +353,24 @@ fn lex3dot(
};
let t = match (q) {
r: rune => switch (r) {
- '.' => return (btoken::ELLIPSIS: token, loc),
- * => btoken::SLICE,
+ '.' => return (ltok::ELLIPSIS, void, loc),
+ * => ltok::SLICE,
},
- io::EOF => btoken::SLICE,
+ io::EOF => ltok::SLICE,
};
unget(lex, q);
t;
},
* => {
unget(lex, n);
- btoken::DOT;
+ ltok::DOT;
}
};
- return (tok, loc);
+ return (tok, void, loc);
};
-fn lex3lt(
- lex: *lexer,
- loc: location,
- n: rune,
-) ((token, location) | io::EOF | error) = {
- let tok: token = switch (n) {
+fn lex3lt(lex: *lexer, loc: location, n: rune) (token | error) = {
+ let tok: ltok = switch (n) {
'<' => {
let q = match (next(lex)?) {
io::EOF => io::EOF,
@@ -405,29 +378,25 @@ fn lex3lt(
};
let t = match (q) {
r: rune => switch (r) {
- '=' => return (btoken::LSHIFTEQ: token, loc),
- * => btoken::LSHIFT,
+ '=' => return (ltok::LSHIFTEQ, void, loc),
+ * => ltok::LSHIFT,
},
- io::EOF => btoken::LSHIFT,
+ io::EOF => ltok::LSHIFT,
};
unget(lex, q);
t;
},
- '=' => btoken::LESSEQ,
+ '=' => ltok::LESSEQ,
* => {
unget(lex, n);
- btoken::LESS;
+ ltok::LESS;
}
};
- return (tok, loc);
+ return (tok, void, loc);
};
-fn lex3gt(
- lex: *lexer,
- loc: location,
- n: rune,
-) ((token, location) | io::EOF | error) = {
- let tok: token = switch (n) {
+fn lex3gt(lex: *lexer, loc: location, n: rune) (token | error) = {
+ let tok: ltok = switch (n) {
'>' => {
let q = match (next(lex)?) {
io::EOF => io::EOF,
@@ -435,27 +404,27 @@ fn lex3gt(
};
let t = match (q) {
r: rune => switch (r) {
- '=' => return (btoken::RSHIFTEQ: token, loc),
- * => btoken::RSHIFT,
+ '=' => return (ltok::RSHIFTEQ, void, loc),
+ * => ltok::RSHIFT,
},
- io::EOF => btoken::RSHIFT,
+ io::EOF => ltok::RSHIFT,
};
unget(lex, q);
t;
},
- '=' => btoken::GREATEREQ,
+ '=' => ltok::GREATEREQ,
* => {
unget(lex, n);
- btoken::GREATER;
+ ltok::GREATER;
}
};
- return (tok, loc);
+ return (tok, void, loc);
};
-// Unlex a single token. The next call to [lex] will return this token, location
-// pair. Only one unlex is supported at a time; you must call [lex] before
-// calling [unlex] again.
-export fn unlex(lex: *lexer, tok: (token, location)) void = {
+// Unlex a single token. The next call to [lex] will return this token. Only one
+// unlex is supported at a time; you must call [lex] before calling [unlex]
+// again.
+export fn unlex(lex: *lexer, tok: token) void = {
assert(lex.un is void, "attempted to unlex more than one token");
lex.un = tok;
};
diff --git a/hare/lex/token.ha b/hare/lex/token.ha
@@ -2,7 +2,7 @@ use encoding::utf8;
use strings;
// A token with no additional context, such as '+'
-export type btoken = enum {
+export type ltok = enum {
// Keep ordered with bmap
// Alpha sorted
ATTR_FINI,
@@ -117,10 +117,16 @@ export type btoken = enum {
SLICE,
TIMES,
TIMESEQ,
+ LAST_BTOK = TIMESEQ,
+
+ NAME,
+ LITERAL,
+ LABEL,
+ EOF,
};
const bmap: [_]str = [
- // Keep ordered with btoken
+ // Keep ordered with tok
"@fini",
"@init",
"@noreturn",
@@ -232,18 +238,13 @@ const bmap: [_]str = [
"*=",
];
-// A loop label, such as ':example'
-export type label = str;
-
-// A name, such as 'example'
-export type name = str;
-
export type iconst = i64;
export type fconst = f64;
// A token for a literal value, such as '1337u32'
-export type literal = (u8 | u16 | u32 | u64 | uint | uintptr | i8 | i16 | i32 |
- i64 | int | iconst | f32 | f64 | fconst | rune | str);
+// TODO: Refactor this into a union { i64, u64, f64, str } + storage enum
+export type value = (u8 | u16 | u32 | u64 | uint | uintptr | i8 | i16 | i32 |
+ i64 | int | iconst | f32 | f64 | fconst | rune | str | void);
// A location within a source file.
// The path is borrowed from the file name given to the lexer.
@@ -254,30 +255,36 @@ export type location = struct {
};
// A single lexical token.
-export type token = (btoken | label | name | literal);
+export type token = (ltok, value, location);
// Converts a token to its string representation
-export fn tokstr(tok: token) const str = match (tok) {
- b: btoken => bmap[b: int],
- n: name => n: str,
- l: literal => match (l) {
- u8 => "u8",
- u16 => "u16",
- u32 => "u32",
- u64 => "u64",
- uint => "uint",
- uintptr => "uintptr",
- i8 => "i8",
- i16 => "i16",
- i32 => "i32",
- i64 => "i64",
- int => "int",
- iconst => "iconst",
- f32 => "f32",
- f64 => "f64",
- fconst => "fconst",
- rune => "rune",
- str => "str",
- },
- * => abort(), // TODO
+export fn tokstr(tok: token) const str = {
+ if (tok.0 <= ltok::LAST_BTOK) {
+ return bmap[tok.0: int];
+ };
+ return switch (tok.0) {
+ ltok::NAME => tok.1 as str,
+ ltok::LABEL => abort(), // TODO
+ ltok::LITERAL => match (tok.1) {
+ u8 => "u8",
+ u16 => "u16",
+ u32 => "u32",
+ u64 => "u64",
+ uint => "uint",
+ uintptr => "uintptr",
+ i8 => "i8",
+ i16 => "i16",
+ i32 => "i32",
+ i64 => "i64",
+ int => "int",
+ iconst => "iconst",
+ f32 => "f32",
+ f64 => "f64",
+ fconst => "fconst",
+ rune => "rune",
+ str => "str",
+ },
+ ltok::EOF => "EOF",
+ * => abort(),
+ };
};
diff --git a/hare/parse/+test.ha b/hare/parse/+test.ha
@@ -16,7 +16,8 @@ use strio;
defer ast::ident_free(ident);
assert(len(ident) == 1);
assert(ident[0] == "foo");
- assert(lex::lex(&lexer) is io::EOF);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::EOF);
};
{
@@ -27,7 +28,8 @@ use strio;
defer ast::ident_free(ident);
assert(len(ident) == 2);
assert(ident[0] == "foo" && ident[1] == "bar");
- assert(lex::lex(&lexer) is io::EOF);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::EOF);
};
{
@@ -39,7 +41,8 @@ use strio;
assert(len(ident) == 3);
assert(ident[0] == "foo" && ident[1] == "bar"
&& ident[2] == "baz");
- assert(lex::lex(&lexer) is io::EOF);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::EOF);
};
{
@@ -50,8 +53,8 @@ use strio;
defer ast::ident_free(ident);
assert(len(ident) == 2);
assert(ident[0] == "foo" && ident[1] == "bar");
- let tok = lex::lex(&lexer) as (lex::token, lex::location);
- assert(tok.0 as lex::btoken == lex::btoken::SEMICOLON);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::SEMICOLON);
};
};
@@ -70,7 +73,8 @@ use strio;
let mod = mods[0] as ast::import_module;
assert(len(mod) == 1 && mod[0] == "foo");
- assert(lex::lex(&lexer) is io::EOF);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::EOF);
};
{
@@ -98,8 +102,8 @@ use strio;
};
};
- let tok = lex::lex(&lexer) as (lex::token, lex::location);
- assert(tok.0 as lex::btoken == lex::btoken::EXPORT);
+ let tok = lex::lex(&lexer) as lex::token;
+ assert(tok.0 == lex::ltok::EXPORT);
};
{
diff --git a/hare/parse/decl.ha b/hare/parse/decl.ha
@@ -1,67 +1,59 @@
use ascii;
use hare::ast;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
use hare::lex;
use hare::unparse;
use strings;
fn attr_symbol(lexer: *lex::lexer) (str | error) = {
- want_btoken(lexer, btoken::LPAREN)?;
- let s: (str, lex::location) = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Expected string, got EOF"),
- t: (lex::token, lex::location) => match (t.0) {
- l: lex::literal => match (l) {
- s: str => (s, t.1),
- * => return syntaxerr(t.1,
- "Unexpected {}, was expecting string",
- lex::tokstr(t.0)),
- },
- * => return syntaxerr(t.1,
- "Unexpected {}, was expecting string",
- lex::tokstr(t.0)),
- },
+ want_tok(lexer, ltok::LPAREN)?;
+ let t = want_tok(lexer, ltok::LITERAL)?;
+ let s = match (t.1) {
+ s: str => s,
+ * => return syntaxerr(t.2,
+ "Unexpected {}, was expecting string",
+ lex::tokstr(t)),
};
- let d = strings::iter(s.0);
+ let d = strings::iter(s);
match (strings::next(&d)) {
void => void,
- r: rune => synassert(s.1,
+ r: rune => synassert(t.2,
ascii::isalpha(r) || r == '.' || r == '_',
"Invalid symbol")?,
};
for (true) match (strings::next(&d)) {
void => break,
- r: rune => synassert(s.1,
+ r: rune => synassert(t.2,
ascii::isalnum(r) || r == '$' || r == '.' || r == '_',
"Invalid symbol")?,
};
- want_btoken(lexer, btoken::RPAREN)?;
- return s.0;
+ want_tok(lexer, ltok::RPAREN)?;
+ return s;
};
fn decl_global(
lexer: *lex::lexer,
- tok: btoken
+ tok: ltok
) ([]ast::decl_global | error) = {
let decl: []ast::decl_global = [];
for (true) {
- let symbol = if (tok == btoken::CONST || tok == btoken::LET) {
- match (try_btoken(lexer, btoken::ATTR_SYMBOL)?) {
+ let symbol = if (tok == ltok::CONST || tok == ltok::LET) {
+ match (try_tok(lexer, ltok::ATTR_SYMBOL)?) {
void => "",
- lex::btoken => attr_symbol(lexer)?,
+ lex::token => attr_symbol(lexer)?,
};
} else "";
let ident = ident(lexer)?;
- want_btoken(lexer, btoken::COLON)?;
+ want_tok(lexer, ltok::COLON)?;
let _type = _type(lexer)?;
- if (tok == btoken::CONST) {
+ if (tok == ltok::CONST) {
_type.flags |= ast::type_flags::CONST;
};
- want_btoken(lexer, btoken::EQUAL)?;
+ want_tok(lexer, ltok::EQUAL)?;
let init = expression(lexer)?;
- let btok = try_btoken(lexer, btoken::COMMA)?;
+ let btok = try_tok(lexer, ltok::COMMA)?;
append(decl, ast::decl_global {
- is_const = tok == btoken::DEF,
+ is_const = tok == ltok::DEF,
symbol = symbol,
ident = ident,
_type = _type,
@@ -79,9 +71,9 @@ fn decl_type(lexer: *lex::lexer) ([]ast::decl_type | error) = {
let decl: []ast::decl_type = [];
for (true) {
let ident = ident(lexer)?;
- want_btoken(lexer, btoken::EQUAL)?;
+ want_tok(lexer, ltok::EQUAL)?;
let _type = _type(lexer)?;
- let btok = try_btoken(lexer, btoken::COMMA)?;
+ let btok = try_tok(lexer, ltok::COMMA)?;
append(decl, ast::decl_type {
ident = ident,
_type = _type,
@@ -96,23 +88,23 @@ fn decl_type(lexer: *lex::lexer) ([]ast::decl_type | error) = {
fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = {
let attr = ast::fndecl_attrs::NONE, noreturn = false, sym = "";
- let attrs = [
- btoken::ATTR_FINI, btoken::ATTR_INIT, btoken::ATTR_TEST,
- btoken::ATTR_NORETURN, btoken::ATTR_SYMBOL
+ const attrs = [
+ ltok::ATTR_FINI, ltok::ATTR_INIT, ltok::ATTR_TEST,
+ ltok::ATTR_NORETURN, ltok::ATTR_SYMBOL
];
- for (true) match (try_btoken(lexer, attrs...)?) {
+ for (true) match (try_tok(lexer, attrs...)?) {
void => break,
- b: lex::btoken => switch (b) {
- btoken::ATTR_FINI => attr = ast::fndecl_attrs::FINI,
- btoken::ATTR_INIT => attr = ast::fndecl_attrs::INIT,
- btoken::ATTR_TEST => attr = ast::fndecl_attrs::TEST,
- btoken::ATTR_NORETURN => noreturn = true,
- btoken::ATTR_SYMBOL => sym = attr_symbol(lexer)?,
+ t: lex::token => switch (t.0) {
+ ltok::ATTR_FINI => attr = ast::fndecl_attrs::FINI,
+ ltok::ATTR_INIT => attr = ast::fndecl_attrs::INIT,
+ ltok::ATTR_TEST => attr = ast::fndecl_attrs::TEST,
+ ltok::ATTR_NORETURN => noreturn = true,
+ ltok::ATTR_SYMBOL => sym = attr_symbol(lexer)?,
* => abort("unreachable"),
},
};
- want_btoken(lexer, btoken::FN)?;
+ want_tok(lexer, ltok::FN)?;
let ident_loc = mkloc(lexer);
let ident = ident(lexer)?;
let proto_loc = mkloc(lexer);
@@ -121,9 +113,9 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = {
prototype.attrs |= ast::func_attrs::NORETURN;
};
- let tok = want_btoken(lexer, btoken::EQUAL, btoken::SEMICOLON)?;
- let body = switch (tok) {
- btoken::EQUAL => {
+ let tok = want_tok(lexer, ltok::EQUAL, ltok::SEMICOLON)?;
+ let body = switch (tok.0) {
+ ltok::EQUAL => {
synassert(ident_loc, len(ident) == 1,
"Unexpected identifier, was expecting name")?;
const params = prototype.params;
@@ -132,10 +124,9 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = {
len(params[i].name) > 0,
"Expected parameter name in function declaration")?;
};
- compound_expression(lexer)?;
+ expression(lexer)?;
},
- // We don't care about the location
- btoken::SEMICOLON => lex::unlex(lexer, (tok, proto_loc)),
+ ltok::SEMICOLON => lex::unlex(lexer, tok),
};
return ast::decl_func {
@@ -155,40 +146,24 @@ fn decl_func(lexer: *lex::lexer) (ast::decl_func | error) = {
export fn decls(lexer: *lex::lexer) ([]ast::decl | error) = {
let decls: []ast::decl = [];
for (true) {
- match (lex::lex(lexer)?) {
- io::EOF => break,
- t: (lex::token, lex::location) => lex::unlex(lexer, t),
- };
- let exported = match (try_btoken(lexer, btoken::EXPORT)?) {
+ if (peek_tok(lexer, ltok::EOF)? is lex::token) break;
+ let exported = match (try_tok(lexer, ltok::EXPORT)?) {
void => false,
- lex::btoken => true,
- };
- let tok = match(lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Unexpected EOF, was expecting declaration"),
- t: (lex::token, lex::location) => t,
- };
- let btok = match (tok.0) {
- b: lex::btoken => b,
- * => return syntaxerr(mkloc(lexer),
- "Unexpected {}, was expecting declaration",
- lex::tokstr(tok.0)),
+ lex::token => true,
};
- let decl = switch (btok) {
- btoken::CONST, btoken::LET, btoken::DEF =>
- decl_global(lexer, btok)?,
- btoken::TYPE => decl_type(lexer)?,
- * => {
- lex::unlex(lexer, tok);
- decl_func(lexer)?;
- },
+ const toks = [ltok::CONST, ltok::LET, ltok::DEF, ltok::TYPE];
+ let decl = match (try_tok(lexer, toks...)?) {
+ void => decl_func(lexer)?,
+ t: lex::token =>
+ if (t.0 == ltok::TYPE) decl_type(lexer)?
+ else decl_global(lexer, t.0)?,
};
append(decls, ast::decl {
exported = exported,
loc = mkloc(lexer),
decl = decl,
});
- want_btoken(lexer, btoken::SEMICOLON)?;
+ want_tok(lexer, ltok::SEMICOLON)?;
};
return decls;
};
diff --git a/hare/parse/expr.ha b/hare/parse/expr.ha
@@ -1,5 +1,5 @@
use hare::ast;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
use hare::lex;
fn binarithm(
@@ -14,19 +14,19 @@ fn binarithm(
expr: ast::expr => expr,
};
- let tok = mustlex(lexer)?;
+ let tok = lex::lex(lexer)?;
let j = precedence(tok);
for (j >= i; j = precedence(tok)) {
const op = binop_for_tok(tok);
let rvalue = cast(lexer, void)?;
- tok = mustlex(lexer)?;
+ tok = lex::lex(lexer)?;
let k = precedence(tok);
for (k > j; k = precedence(tok)) {
lex::unlex(lexer, tok);
rvalue = binarithm(lexer, rvalue, k)?;
- tok = mustlex(lexer)?;
+ tok = lex::lex(lexer)?;
};
let expr = ast::binarithm_expr {
@@ -46,15 +46,15 @@ fn cast(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = {
void => unarithm(lexer)?,
e: ast::expr => e,
};
- const tok = match (try_btoken(lexer, btoken::COLON,
- btoken::AS, btoken::IS)?) {
+ const tok = match (try_tok(lexer, ltok::COLON,
+ ltok::AS, ltok::IS)?) {
void => return lvalue,
- tok: btoken => tok,
+ tok: lex::token => tok.0,
};
const kind = switch (tok) {
- btoken::COLON => ast::cast_kind::CAST,
- btoken::AS => ast::cast_kind::ASSERTION,
- btoken::IS => ast::cast_kind::TEST,
+ ltok::COLON => ast::cast_kind::CAST,
+ ltok::AS => ast::cast_kind::ASSERTION,
+ ltok::IS => ast::cast_kind::TEST,
* => abort(),
};
return cast(lexer, ast::cast_expr {
@@ -65,7 +65,7 @@ fn cast(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = {
};
fn constant(lexer: *lex::lexer) (ast::expr | error) = {
- want_btoken(lexer, btoken::VOID)?;
+ want_tok(lexer, ltok::VOID)?;
return void;
};
@@ -78,45 +78,29 @@ fn objsel(lexer: *lex::lexer) (ast::expr | error) = {
};
fn plain_expression(lexer: *lex::lexer) (ast::expr | error) = {
- let tok = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Unexpected EOF, was expecting an expression"),
- tok: (lex::token, lex::location) => tok,
- };
- let tok = match (tok.0) {
- btok: btoken => {
- lex::unlex(lexer, tok);
- btok;
- },
- lit: lex::literal => {
- lex::unlex(lexer, tok);
- return constant(lexer);
+ let tok = peek_tok(lexer)? as lex::token;
+ return switch (tok.0) {
+ ltok::TRUE,
+ ltok::FALSE,
+ ltok::NULL,
+ ltok::VOID => constant(lexer),
+ ltok::LBRACKET => abort(), // TODO: Array literal
+ ltok::STRUCT => abort(), // TODO: Struct literal
+ ltok::LPAREN => {
+ let ex = expression(lexer);
+ return switch (want_tok(lexer,
+ ltok::RPAREN, ltok::COMMA)?.0) {
+ ltok::RPAREN => ex,
+ ltok::COMMA => abort(), // TODO: Tuple literal
+ * => abort(),
+ };
},
- name: lex::name => {
- lex::unlex(lexer, tok);
+ ltok::LITERAL => return constant(lexer),
+ ltok::NAME => {
let id = ident(lexer)?;
- return match (try_btoken(lexer, btoken::LBRACE)?) {
+ return match (try_tok(lexer, ltok::LBRACE)?) {
void => id: ast::access_identifier,
- btoken => abort(), // TODO: Struct literal
- };
- },
- };
- return switch (tok) {
- btoken::TRUE,
- btoken::FALSE,
- btoken::NULL,
- btoken::VOID => {
- constant(lexer);
- },
- btoken::LBRACKET => abort(), // TODO: Array literal
- btoken::STRUCT => abort(), // TODO: Struct literal
- btoken::LPAREN => {
- let ex = expression(lexer);
- return switch (want_btoken(lexer,
- btoken::RPAREN, btoken::COMMA)?) {
- btoken::RPAREN => ex,
- btoken::COMMA => abort(), // TODO: Tuple literal
- * => abort(),
+ lex::token => abort(), // TODO: Struct literal
};
},
* => syntaxerr(mkloc(lexer),
@@ -127,12 +111,12 @@ fn plain_expression(lexer: *lex::lexer) (ast::expr | error) = {
fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) = {
// Built-ins (XXX: these should probably be moved in the Hare grammar)
- match (peek_btoken(lexer,
- btoken::ALLOC, btoken::APPEND, btoken::FREE,
- btoken::DELETE, btoken::ABORT, btoken::ASSERT,
- btoken::STATIC, btoken::SIZE, btoken::LEN,
- btoken::OFFSET)?) {
- tok: btoken => {
+ match (peek_tok(lexer,
+ ltok::ALLOC, ltok::APPEND, ltok::FREE,
+ ltok::DELETE, ltok::ABORT, ltok::ASSERT,
+ ltok::STATIC, ltok::SIZE, ltok::LEN,
+ ltok::OFFSET)?) {
+ tok: lex::token => {
if (lvalue is ast::expr) {
return syntaxerr(mkloc(lexer),
"Unexpected {}, was expecting '(', '.', or '['",
@@ -148,13 +132,13 @@ fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) =
ex: ast::expr => ex,
};
- match (try_btoken(lexer, btoken::LPAREN, btoken::DOT, btoken::LBRACKET,
- btoken::QUESTION)) {
- tok: btoken => switch (tok) {
- btoken::LPAREN => abort(), // TODO: Calls
- btoken::DOT => abort(), // TODO: Field access
- btoken::LBRACKET => abort(), // TODO: Indexing
- btoken::QUESTION => abort(), // TODO: Propagation
+ match (try_tok(lexer, ltok::LPAREN, ltok::DOT, ltok::LBRACKET,
+ ltok::QUESTION)) {
+ tok: lex::token => switch (tok.0) {
+ ltok::LPAREN => abort(), // TODO: Calls
+ ltok::DOT => abort(), // TODO: Field access
+ ltok::LBRACKET => abort(), // TODO: Indexing
+ ltok::QUESTION => abort(), // TODO: Propagation
* => abort(),
},
void => return lvalue,
@@ -164,23 +148,23 @@ fn postfix(lexer: *lex::lexer, lvalue: (ast::expr | void)) (ast::expr | error) =
};
fn unarithm(lexer: *lex::lexer) (ast::expr | error) = {
- const tok = match (try_btoken(lexer,
- btoken::PLUS, btoken::MINUS, btoken::BNOT,
- btoken::LNOT, btoken::TIMES, btoken::BAND)) {
+ const tok = match (try_tok(lexer,
+ ltok::PLUS, ltok::MINUS, ltok::BNOT,
+ ltok::LNOT, ltok::TIMES, ltok::BAND)) {
void => return postfix(lexer, void),
- tok: btoken => tok,
+ tok: lex::token => tok.0,
};
const op = switch (tok) {
- btoken::PLUS => ast::unarithm_op::PLUS,
- btoken::MINUS => ast::unarithm_op::MINUS,
- btoken::BNOT => ast::unarithm_op::BNOT,
- btoken::LNOT => ast::unarithm_op::LNOT,
- btoken::TIMES => ast::unarithm_op::DEREF,
- btoken::BAND => ast::unarithm_op::ADDR,
+ ltok::PLUS => ast::unarithm_op::PLUS,
+ ltok::MINUS => ast::unarithm_op::MINUS,
+ ltok::BNOT => ast::unarithm_op::BNOT,
+ ltok::LNOT => ast::unarithm_op::LNOT,
+ ltok::TIMES => ast::unarithm_op::DEREF,
+ ltok::BAND => ast::unarithm_op::ADDR,
* => abort(),
};
let operand =
- if (tok == btoken::BAND) objsel(lexer)?
+ if (tok == ltok::BAND) objsel(lexer)?
else unarithm(lexer)?;
return ast::unarithm_expr {
op = op,
@@ -188,87 +172,71 @@ fn unarithm(lexer: *lex::lexer) (ast::expr | error) = {
};
};
-fn binop_for_tok(tok: (lex::token, lex::location)) ast::binarithm_op = {
- let tok = match (tok.0) {
- b: btoken => b,
- * => abort(),
- };
- return switch (tok) {
- btoken::BAND => ast::binarithm_op::BAND,
- btoken::BOR => ast::binarithm_op::BOR,
- btoken::BXOR => ast::binarithm_op::BXOR,
- btoken::DIV => ast::binarithm_op::DIV,
- btoken::GREATER => ast::binarithm_op::GT,
- btoken::GREATEREQ => ast::binarithm_op::GTEQ,
- btoken::LAND => ast::binarithm_op::LAND,
- btoken::LEQUAL => ast::binarithm_op::LEQUAL,
- btoken::LESS => ast::binarithm_op::LESS,
- btoken::LESSEQ => ast::binarithm_op::LESSEQ,
- btoken::LOR => ast::binarithm_op::LOR,
- btoken::LSHIFT => ast::binarithm_op::LSHIFT,
- btoken::LXOR => ast::binarithm_op::LXOR,
- btoken::MINUS => ast::binarithm_op::MINUS,
- btoken::MODULO => ast::binarithm_op::MODULO,
- btoken::NEQUAL => ast::binarithm_op::NEQUAL,
- btoken::PLUS => ast::binarithm_op::PLUS,
- btoken::RSHIFT => ast::binarithm_op::RSHIFT,
- btoken::TIMES => ast::binarithm_op::TIMES,
- * => abort(),
- };
+fn binop_for_tok(tok: lex::token) ast::binarithm_op = switch (tok.0) {
+ ltok::BAND => ast::binarithm_op::BAND,
+ ltok::BOR => ast::binarithm_op::BOR,
+ ltok::BXOR => ast::binarithm_op::BXOR,
+ ltok::DIV => ast::binarithm_op::DIV,
+ ltok::GREATER => ast::binarithm_op::GT,
+ ltok::GREATEREQ => ast::binarithm_op::GTEQ,
+ ltok::LAND => ast::binarithm_op::LAND,
+ ltok::LEQUAL => ast::binarithm_op::LEQUAL,
+ ltok::LESS => ast::binarithm_op::LESS,
+ ltok::LESSEQ => ast::binarithm_op::LESSEQ,
+ ltok::LOR => ast::binarithm_op::LOR,
+ ltok::LSHIFT => ast::binarithm_op::LSHIFT,
+ ltok::LXOR => ast::binarithm_op::LXOR,
+ ltok::MINUS => ast::binarithm_op::MINUS,
+ ltok::MODULO => ast::binarithm_op::MODULO,
+ ltok::NEQUAL => ast::binarithm_op::NEQUAL,
+ ltok::PLUS => ast::binarithm_op::PLUS,
+ ltok::RSHIFT => ast::binarithm_op::RSHIFT,
+ ltok::TIMES => ast::binarithm_op::TIMES,
+ * => abort(),
};
-fn precedence(tok: (lex::token, lex::location)) int = {
- let tok = match (tok.0) {
- b: btoken => b,
- * => return -1,
- };
- return switch (tok) {
- btoken::LOR => 0,
- btoken::LXOR => 1,
- btoken::LAND => 2,
- btoken::LEQUAL,
- btoken::NEQUAL => 3,
- btoken::LESS,
- btoken::LESSEQ,
- btoken::GREATER,
- btoken::GREATEREQ => 4,
- btoken::BOR => 5,
- btoken::BXOR => 6,
- btoken::BAND => 7,
- btoken::LSHIFT,
- btoken::RSHIFT => 8,
- btoken::PLUS,
- btoken::MINUS => 9,
- btoken::TIMES,
- btoken::DIV,
- btoken::MODULO => 10,
- * => -1,
- };
+fn precedence(tok: lex::token) int = switch (tok.0) {
+ ltok::LOR => 0,
+ ltok::LXOR => 1,
+ ltok::LAND => 2,
+ ltok::LEQUAL,
+ ltok::NEQUAL => 3,
+ ltok::LESS,
+ ltok::LESSEQ,
+ ltok::GREATER,
+ ltok::GREATEREQ => 4,
+ ltok::BOR => 5,
+ ltok::BXOR => 6,
+ ltok::BAND => 7,
+ ltok::LSHIFT,
+ ltok::RSHIFT => 8,
+ ltok::PLUS,
+ ltok::MINUS => 9,
+ ltok::TIMES,
+ ltok::DIV,
+ ltok::MODULO => 10,
+ * => -1,
};
fn expression_list(lexer: *lex::lexer) (ast::expr | error) = {
let items: ast::list_expr = [];
for (let more = true; more) {
- let tok = mustlex(lexer)?;
- lex::unlex(lexer, tok);
- let item = match (tok.0) {
- tok: btoken => switch (tok) {
- btoken::RBRACE => break,
- btoken::BREAK,
- btoken::CONTINUE,
- btoken::RETURN => {
- more = false;
- control_statement(lexer)?;
- },
- * => expression(lexer)?,
+ let tok = peek_tok(lexer)? as lex::token;
+ let item = switch (tok.0) {
+ ltok::RBRACE => break,
+ ltok::BREAK,
+ ltok::CONTINUE,
+ ltok::RETURN => {
+ more = false;
+ control_statement(lexer)?;
},
* => expression(lexer)?,
};
append(items, alloc(item));
- want_btoken(lexer, btoken::SEMICOLON)?;
+ want_tok(lexer, ltok::SEMICOLON)?;
};
- want_btoken(lexer, btoken::RBRACE)?;
+ want_tok(lexer, ltok::RBRACE)?;
return items;
};
diff --git a/hare/parse/exprclass.ha b/hare/parse/exprclass.ha
@@ -1,5 +1,5 @@
use hare::ast;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
use hare::lex;
use io;
@@ -8,25 +8,3 @@ export fn expression(lexer: *lex::lexer) (ast::expr | error) = {
// TODO: if, for, switch, match, etc
return binarithm(lexer, void, 0);
};
-
-// Parses a compound-expression.
-export fn compound_expression(lexer: *lex::lexer) (ast::expr | error) = {
- let tok = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Unexpected EOF, expected compound expression"),
- t: (lex::token, lex::location) => t,
- };
-
- lex::unlex(lexer, tok);
- let tok = match (tok.0) {
- tok: btoken => tok,
- * => return expression(lexer),
- };
-
- return switch (tok) {
- btoken::LBRACE => expression_list(lexer),
- btoken::BREAK, btoken::CONTINUE, btoken::RETURN =>
- control_statement(lexer),
- * => expression(lexer),
- };
-};
diff --git a/hare/parse/ident.ha b/hare/parse/ident.ha
@@ -1,18 +1,18 @@
use hare::ast;
use hare::lex;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
fn ident_trailing(lexer: *lex::lexer) ((ast::ident, bool) | error) = {
let ident: []str = [];
let z = 0z;
for (true) {
- let name = match (try_name(lexer)?) {
- n: lex::name => n,
+ let name = match (try_tok(lexer, ltok::NAME)?) {
+ t: lex::token => t.1 as str,
void => return (ident: ast::ident, true),
};
- append(ident, name: str);
+ append(ident, name);
z += len(name);
- match (try_btoken(lexer, btoken::DOUBLE_COLON)?) {
+ match (try_tok(lexer, ltok::DOUBLE_COLON)?) {
void => break,
* => void, // Grab the next ident
};
diff --git a/hare/parse/import.ha b/hare/parse/import.ha
@@ -1,17 +1,17 @@
use hare::ast;
use hare::lex;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
fn name_list(lexer: *lex::lexer) ([]str | error) = {
let names: []str = [];
for (true) {
- append(names, want_name(lexer)?: str);
- switch (want_btoken(lexer, btoken::COMMA, btoken::RBRACE)?) {
- btoken::COMMA => match (try_btoken(lexer, btoken::RBRACE)?) {
+ append(names, want_tok(lexer, ltok::NAME)?.1 as str);
+ switch (want_tok(lexer, ltok::COMMA, ltok::RBRACE)?.0) {
+ ltok::COMMA => match (try_tok(lexer, ltok::RBRACE)?) {
void => void,
* => return names,
},
- btoken::RBRACE => return names,
+ ltok::RBRACE => return names,
* => abort(), // Unreachable
};
};
@@ -22,21 +22,21 @@ fn name_list(lexer: *lex::lexer) ([]str | error) = {
export fn imports(lexer: *lex::lexer) ([]ast::import | error) = {
let imports: []ast::import = [];
for (true) {
- match (try_btoken(lexer, btoken::USE)?) {
+ match (try_tok(lexer, ltok::USE)?) {
void => break,
* => void,
};
let name = ident_trailing(lexer)?;
- switch (want_btoken(lexer, btoken::SEMICOLON, btoken::LBRACE,
- btoken::EQUAL)?) {
- btoken::SEMICOLON => {
+ switch (want_tok(lexer, ltok::SEMICOLON, ltok::LBRACE,
+ ltok::EQUAL)?.0) {
+ ltok::SEMICOLON => {
synassert(mkloc(lexer), !name.1,
"Unexpected trailing :: in ident")?;
append(imports, name.0: ast::import_module);
},
- btoken::LBRACE => {
+ ltok::LBRACE => {
synassert(mkloc(lexer), name.1,
"Expected trailing :: in ident")?;
let objects = name_list(lexer)?;
@@ -44,9 +44,9 @@ export fn imports(lexer: *lex::lexer) ([]ast::import | error) = {
ident = name.0,
objects = objects,
});
- want_btoken(lexer, btoken::SEMICOLON)?;
+ want_tok(lexer, ltok::SEMICOLON)?;
},
- btoken::EQUAL => {
+ ltok::EQUAL => {
synassert(mkloc(lexer),
len(name.0) == 1 && !name.1,
"Expected name, not ident")?;
@@ -55,7 +55,7 @@ export fn imports(lexer: *lex::lexer) ([]ast::import | error) = {
ident = ident,
alias = name.0[0],
});
- want_btoken(lexer, btoken::SEMICOLON)?;
+ want_tok(lexer, ltok::SEMICOLON)?;
},
* => abort(), // Unreachable
};
diff --git a/hare/parse/type.ha b/hare/parse/type.ha
@@ -1,50 +1,50 @@
use hare::ast;
use hare::ast::{builtin_type};
use hare::lex;
-use hare::lex::{btoken};
+use hare::lex::{ltok};
fn prototype(lexer: *lex::lexer) (ast::func_type | error) = {
let variadism = ast::variadism::NONE;
let params: []ast::func_param = [];
- want_btoken(lexer, btoken::LPAREN)?;
- for (try_btoken(lexer, btoken::RPAREN)? is void) {
+ want_tok(lexer, ltok::LPAREN)?;
+ for (try_tok(lexer, ltok::RPAREN)? is void) {
let loc = mkloc(lexer);
- match (try_btoken(lexer, btoken::ELLIPSIS)?) {
+ match (try_tok(lexer, ltok::ELLIPSIS)?) {
void => void,
- lex::btoken => {
+ lex::token => {
synassert(loc, len(params) > 0,
"Expected at least one non-variadic parameter for C-style variadism")?;
variadism = ast::variadism::C;
- try_btoken(lexer, btoken::COMMA)?;
- want_btoken(lexer, btoken::RPAREN)?;
+ try_tok(lexer, ltok::COMMA)?;
+ want_tok(lexer, ltok::RPAREN)?;
break;
},
};
- let name = match (try_btoken(lexer, btoken::UNDERSCORE)?) {
- void => want_name(lexer)?: str,
- lex::btoken => "",
+ let name = match (try_tok(lexer, ltok::UNDERSCORE)?) {
+ void => want_tok(lexer, ltok::NAME)?.1 as str,
+ lex::token => "",
};
- want_btoken(lexer, btoken::COLON);
+ want_tok(lexer, ltok::COLON);
append(params, ast::func_param {
loc = loc,
name = name,
_type = alloc(_type(lexer)?),
});
- match (try_btoken(lexer, btoken::ELLIPSIS)?) {
+ match (try_tok(lexer, ltok::ELLIPSIS)?) {
void => void,
- lex::btoken => {
+ lex::token => {
variadism = ast::variadism::HARE;
- try_btoken(lexer, btoken::COMMA)?;
- want_btoken(lexer, btoken::RPAREN)?;
+ try_tok(lexer, ltok::COMMA)?;
+ want_tok(lexer, ltok::RPAREN)?;
break;
},
};
- match (try_btoken(lexer, btoken::COMMA)?) {
+ match (try_tok(lexer, ltok::COMMA)?) {
void => {
- want_btoken(lexer, btoken::RPAREN)?;
+ want_tok(lexer, ltok::RPAREN)?;
break;
},
- lex::btoken => void,
+ lex::token => void,
};
};
let t = _type(lexer)?;
@@ -58,53 +58,45 @@ fn prototype(lexer: *lex::lexer) (ast::func_type | error) = {
fn integer_type(
lexer: *lex::lexer,
-) (builtin_type | error) = switch (want_btoken(lexer)?) {
- btoken::CHAR => builtin_type::CHAR,
- btoken::I16 => builtin_type::I16,
- btoken::I32 => builtin_type::I32,
- btoken::I64 => builtin_type::I64,
- btoken::I64 => builtin_type::I64,
- btoken::I8 => builtin_type::I8,
- btoken::INT => builtin_type::INT,
- btoken::SIZE => builtin_type::SIZE,
- btoken::U16 => builtin_type::U16,
- btoken::U32 => builtin_type::U32,
- btoken::U64 => builtin_type::U64,
- btoken::U64 => builtin_type::U64,
- btoken::U8 => builtin_type::U8,
- btoken::UINT => builtin_type::UINT,
- btoken::UINTPTR => builtin_type::UINTPTR,
+) (builtin_type | error) = switch (want_tok(lexer)?.0) {
+ ltok::CHAR => builtin_type::CHAR,
+ ltok::I16 => builtin_type::I16,
+ ltok::I32 => builtin_type::I32,
+ ltok::I64 => builtin_type::I64,
+ ltok::I64 => builtin_type::I64,
+ ltok::I8 => builtin_type::I8,
+ ltok::INT => builtin_type::INT,
+ ltok::SIZE => builtin_type::SIZE,
+ ltok::U16 => builtin_type::U16,
+ ltok::U32 => builtin_type::U32,
+ ltok::U64 => builtin_type::U64,
+ ltok::U64 => builtin_type::U64,
+ ltok::U8 => builtin_type::U8,
+ ltok::UINT => builtin_type::UINT,
+ ltok::UINTPTR => builtin_type::UINTPTR,
* => syntaxerr(mkloc(lexer), "Expected integer type"),
};
fn primitive_type(lexer: *lex::lexer) (ast::_type | error) = {
- let t = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Expected primitive type, got EOF"),
- t: (lex::token, lex::location) => t,
- };
- let b = match (t.0) {
- b: lex::btoken => b,
- * => return syntaxerr(mkloc(lexer),
- "Unexpected {}, was expecting primitive type",
- lex::tokstr(t.0)),
- };
- let builtin = switch (b) {
- btoken::CHAR, btoken::I16, btoken::I32, btoken::I64,
- btoken::I64, btoken::I8, btoken::INT, btoken::SIZE, btoken::U16,
- btoken::U32, btoken::U64, btoken::U64, btoken::U8, btoken::UINT,
- btoken::UINTPTR => {
- lex::unlex(lexer, t);
+ let tok = want_tok(lexer)?;
+ let builtin = switch (tok.0) {
+ ltok::CHAR, ltok::I16, ltok::I32, ltok::I64,
+ ltok::I64, ltok::I8, ltok::INT, ltok::SIZE, ltok::U16,
+ ltok::U32, ltok::U64, ltok::U64, ltok::U8, ltok::UINT,
+ ltok::UINTPTR => {
+ lex::unlex(lexer, tok);
integer_type(lexer)?;
},
- btoken::RUNE => builtin_type::RUNE,
- btoken::STR => builtin_type::STR,
- btoken::F32 => builtin_type::F32,
- btoken::F64 => builtin_type::F64,
- btoken::BOOL => builtin_type::BOOL,
- btoken::VOID => builtin_type::VOID,
- btoken::NULL => builtin_type::NULL,
- * => return syntaxerr(mkloc(lexer), "Expected primitive type"),
+ ltok::RUNE => builtin_type::RUNE,
+ ltok::STR => builtin_type::STR,
+ ltok::F32 => builtin_type::F32,
+ ltok::F64 => builtin_type::F64,
+ ltok::BOOL => builtin_type::BOOL,
+ ltok::VOID => builtin_type::VOID,
+ ltok::NULL => builtin_type::NULL,
+ * => return syntaxerr(mkloc(lexer),
+ "Unexected {}, was expecting primitive type",
+ lex::tokstr(tok)),
};
return ast::_type {
loc = mkloc(lexer),
@@ -115,7 +107,7 @@ fn primitive_type(lexer: *lex::lexer) (ast::_type | error) = {
fn alias_type(lexer: *lex::lexer) (ast::_type | error) = {
let loc = mkloc(lexer);
- let unwrap = match (try_btoken(lexer, btoken::ELLIPSIS)?) {
+ let unwrap = match (try_tok(lexer, ltok::ELLIPSIS)?) {
void => false,
* => true,
};
@@ -131,11 +123,11 @@ fn alias_type(lexer: *lex::lexer) (ast::_type | error) = {
fn pointer_type(lexer: *lex::lexer) (ast::_type | error) = {
let loc = mkloc(lexer);
- let flags = match (try_btoken(lexer, btoken::NULLABLE)?) {
+ let flags = match (try_tok(lexer, ltok::NULLABLE)?) {
void => 0: ast::pointer_flags,
* => ast::pointer_flags::NULLABLE,
};
- want_btoken(lexer, btoken::TIMES)?;
+ want_tok(lexer, ltok::TIMES)?;
return ast::_type {
loc = loc,
flags = 0,
@@ -150,14 +142,14 @@ fn tagged_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = {
let loc = mkloc(lexer);
let tagged: ast::tagged_type = [];
append(tagged, alloc(first));
- for (try_btoken(lexer, btoken::RPAREN)? is void) {
+ for (try_tok(lexer, ltok::RPAREN)? is void) {
append(tagged, alloc(_type(lexer)?));
- match (try_btoken(lexer, btoken::BOR)?) {
+ match (try_tok(lexer, ltok::BOR)?) {
void => {
- want_btoken(lexer, btoken::RPAREN)?;
+ want_tok(lexer, ltok::RPAREN)?;
break;
},
- lex::btoken => void,
+ lex::token => void,
};
};
return ast::_type {
@@ -171,14 +163,14 @@ fn tuple_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = {
let loc = mkloc(lexer);
let tuple: ast::tuple_type = [];
append(tuple, alloc(first));
- for (try_btoken(lexer, btoken::RPAREN)? is void) {
+ for (try_tok(lexer, ltok::RPAREN)? is void) {
append(tuple, alloc(_type(lexer)?));
- match (try_btoken(lexer, btoken::COMMA)?) {
+ match (try_tok(lexer, ltok::COMMA)?) {
void => {
- want_btoken(lexer, btoken::RPAREN)?;
+ want_tok(lexer, ltok::RPAREN)?;
break;
},
- lex::btoken => void,
+ lex::token => void,
};
};
return ast::_type {
@@ -190,11 +182,11 @@ fn tuple_type(lexer: *lex::lexer, first: ast::_type) (ast::_type | error) = {
fn fn_type(lexer: *lex::lexer) (ast::_type | error) = {
let loc = mkloc(lexer);
- let attrs = match (try_btoken(lexer, btoken::ATTR_NORETURN)?) {
+ let attrs = match (try_tok(lexer, ltok::ATTR_NORETURN)?) {
void => 0: ast::func_attrs,
* => ast::func_attrs::NORETURN,
};
- want_btoken(lexer, btoken::FN)?;
+ want_tok(lexer, ltok::FN)?;
let proto = prototype(lexer)?;
proto.attrs |= attrs;
return ast::_type {
@@ -206,49 +198,40 @@ fn fn_type(lexer: *lex::lexer) (ast::_type | error) = {
// Parses a type
export fn _type(lexer: *lex::lexer) (ast::_type | error) = {
- let flags: ast::type_flags = match (try_btoken(lexer, btoken::CONST)?) {
+ let flags: ast::type_flags = match (try_tok(lexer, ltok::CONST)?) {
void => 0,
* => ast::type_flags::CONST,
};
- let t = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Unexpected EOF, was expecting type"),
- t: (lex::token, lex::location) => t,
- };
- lex::unlex(lexer, t);
- let typ: ast::_type = match (t.0) {
- b: lex::btoken => switch (b) {
- btoken::CHAR, btoken::I16, btoken::I32, btoken::I64,
- btoken::I64, btoken::I8, btoken::INT, btoken::SIZE,
- btoken::U16, btoken::U32, btoken::U64, btoken::U64,
- btoken::U8, btoken::UINT, btoken::UINTPTR, btoken::RUNE,
- btoken::STR, btoken::F32, btoken::F64, btoken::BOOL,
- btoken::VOID, btoken::NULL => primitive_type(lexer)?,
- btoken::ENUM => abort(), // TODO
- btoken::NULLABLE, btoken::TIMES => pointer_type(lexer)?,
- btoken::STRUCT, btoken::UNION => abort(), // TODO
- btoken::LBRACKET => abort(), // TODO
- btoken::LPAREN => {
- want_btoken(lexer, btoken::LPAREN)?;
- let t = _type(lexer)?;
- switch (want_btoken(lexer, btoken::BOR,
- btoken::COMMA)?) {
- btoken::BOR => tagged_type(lexer, t)?,
- btoken::COMMA => tuple_type(lexer, t)?,
- * => abort("unreachable"),
- };
- },
- btoken::ATTR_NORETURN, btoken::FN => fn_type(lexer)?,
- btoken::ELLIPSIS => alias_type(lexer)?,
- * => return syntaxerr(mkloc(lexer), "Expected type"),
+ let tok = peek_tok(lexer)? as lex::token;
+ let typ: ast::_type = switch (tok.0) {
+ ltok::CHAR, ltok::I16, ltok::I32, ltok::I64,
+ ltok::I64, ltok::I8, ltok::INT, ltok::SIZE,
+ ltok::U16, ltok::U32, ltok::U64, ltok::U64,
+ ltok::U8, ltok::UINT, ltok::UINTPTR, ltok::RUNE,
+ ltok::STR, ltok::F32, ltok::F64, ltok::BOOL,
+ ltok::VOID, ltok::NULL => primitive_type(lexer)?,
+ ltok::ENUM => abort(), // TODO
+ ltok::NULLABLE, ltok::TIMES => pointer_type(lexer)?,
+ ltok::STRUCT, ltok::UNION => abort(), // TODO
+ ltok::LBRACKET => abort(), // TODO
+ ltok::LPAREN => {
+ want_tok(lexer, ltok::LPAREN)?;
+ let t = _type(lexer)?;
+ switch (want_tok(lexer, ltok::BOR,
+ ltok::COMMA)?.0) {
+ ltok::BOR => tagged_type(lexer, t)?,
+ ltok::COMMA => tuple_type(lexer, t)?,
+ * => abort("unreachable"),
+ };
},
- lex::name => alias_type(lexer)?,
+ ltok::ATTR_NORETURN, ltok::FN => fn_type(lexer)?,
+ ltok::ELLIPSIS, ltok::NAME => alias_type(lexer)?,
* => return syntaxerr(mkloc(lexer),
"Unexpected {}, was expecting type",
- lex::tokstr(t.0)),
+ lex::tokstr(tok)),
};
- match (try_btoken(lexer, btoken::LNOT)?) {
+ match (try_tok(lexer, ltok::LNOT)?) {
void => void,
* => flags |= ast::type_flags::ERROR,
};
diff --git a/hare/parse/util.ha b/hare/parse/util.ha
@@ -4,61 +4,23 @@ use hare::lex;
use io;
use strio;
-// Requires the next token to be a name. Returns that name, or an error.
-fn want_name(lexer: *lex::lexer) (lex::name | error) = {
- match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer),
- "Unexpected EOF, was expecting name"),
- t: (lex::token, lex::location) => match (t.0) {
- n: lex::name => return n,
- * => return syntaxerr(mkloc(lexer),
- "Unexpected {}, was expecting name",
- lex::tokstr(t.0)),
- },
+// Requires the next token to have a matching ltok. Returns that token, or an
+// error.
+fn want_tok(lexer: *lex::lexer, want: lex::ltok...) (lex::token | error) = {
+ let tok = lex::lex(lexer)?;
+ if (len(want) == 0) {
+ return tok;
};
-};
-
-// Looks for a matching name from the lexer, and if not present, unlexes the
-// token and returns void. If found, the token is consumed from the lexer and is
-// returned.
-fn try_name(lexer: *lex::lexer) (lex::name | error | void) = {
- let tuple = match (lex::lex(lexer)?) {
- io::EOF => return,
- t: (lex::token, lex::location) => match (t.0) {
- n: lex::name => return n,
- * => t,
- },
- };
- lex::unlex(lexer, tuple);
-};
-
-// Requires the next token to be a name. Returns that name, or an error.
-fn want_btoken(
- lexer: *lex::lexer,
- want: lex::btoken...
-) (lex::btoken | error) = {
- let tok: lex::token = match (lex::lex(lexer)?) {
- io::EOF => return syntaxerr(mkloc(lexer), "Unexpected EOF"),
- t: (lex::token, lex::location) => match (t.0) {
- b: lex::btoken => {
- if (len(want) == 0) {
- return b;
- };
- for (let i = 0z; i < len(want); i += 1) {
- if (b == want[i]) {
- return b;
- };
- };
- t.0;
- },
- * => t.0,
- },
+ for (let i = 0z; i < len(want); i += 1) {
+ if (tok.0 == want[i]) {
+ return tok;
+ };
};
let buf = strio::dynamic();
defer io::close(buf);
for (let i = 0z; i < len(want); i += 1) {
- fmt::fprintf(buf, lex::tokstr(want[i]));
+ fmt::fprintf(buf, lex::tokstr((want[i], void, mkloc(lexer))));
if (i + 1 < len(want)) {
fmt::fprint(buf, ", ");
};
@@ -67,64 +29,38 @@ fn want_btoken(
lex::tokstr(tok), strio::string(buf));
};
-// Looks for a matching btoken from the lexer, and if not present, unlexes the
+// Looks for a matching ltok from the lexer, and if not present, unlexes the
// token and returns void. If found, the token is consumed from the lexer and is
// returned.
-fn try_btoken(
+fn try_tok(
lexer: *lex::lexer,
- want: lex::btoken...
-) (lex::btoken | error | void) = {
- let tok = lex::lex(lexer);
- let tuple = match (tok?) {
- io::EOF => return,
- t: (lex::token, lex::location) => {
- match (t.0) {
- b: lex::btoken =>
- for (let i = 0z; i < len(want); i += 1) {
- if (b == want[i]) {
- return b;
- };
- },
- * => void,
- };
- t;
- },
+ want: lex::ltok...
+) (lex::token | error | void) = {
+ let tok = lex::lex(lexer)?;
+ assert(len(want) > 0);
+ for (let i = 0z; i < len(want); i += 1) {
+ if (tok.0 == want[i]) {
+ return tok;
+ };
};
- lex::unlex(lexer, tuple);
+ lex::unlex(lexer, tok);
};
-// Looks for a matching btoken from the lexer, unlexes the token, and returns
-// it; or void if it was not a btoken.
-fn peek_btoken(
+// Looks for a matching ltok from the lexer, unlexes the token, and returns
+// it; or void if it was not a ltok.
+fn peek_tok(
lexer: *lex::lexer,
- want: lex::btoken...
-) (lex::btoken | error | void) = {
- let tok = lex::lex(lexer);
- let tuple = match (tok?) {
- io::EOF => return,
- t: (lex::token, lex::location) => {
- match (t.0) {
- b: lex::btoken =>
- for (let i = 0z; i < len(want); i += 1) {
- if (b == want[i]) {
- lex::unlex(lexer, t);
- return b;
- };
- },
- * => void,
- };
- t;
- },
+ want: lex::ltok...
+) (lex::token | error | void) = {
+ let tok = lex::lex(lexer)?;
+ lex::unlex(lexer, tok);
+ if (len(want) == 0) {
+ return tok;
};
- lex::unlex(lexer, tuple);
-};
-
-// Identical to [lex::lex], but considers io::EOF a syntax error.
-fn mustlex(lexer: *lex::lexer) ((lex::token, lex::location) | error) = {
- return match (lex::lex(lexer)) {
- err: lex::error => err,
- io::EOF => syntaxerr(mkloc(lexer), "Unexpected EOF"),
- t: (lex::token, lex::location) => t,
+ for (let i = 0z; i < len(want); i += 1) {
+ if (tok.0 == want[i]) {
+ return tok;
+ };
};
};