hare

[hare] The Hare programming language
git clone https://git.torresjrjr.com/hare.git
Log | Files | Refs | README | LICENSE

token.ha (3929B)


      1 // License: MPL-2.0
      2 // (c) 2021 Bor Grošelj Simić <bor.groseljsimic@telemach.net>
      3 // (c) 2021 Drew DeVault <sir@cmpwn.com>
      4 // (c) 2021 Eyal Sawady <ecs@d2evs.net>
      5 use encoding::utf8;
      6 use strings;
      7 
      8 // A lexical token class.
      9 export type ltok = enum uint {
     10 	// Keep ordered with bmap
     11 	// Alpha sorted
     12 
     13 	ATTR_FINI,
     14 	ATTR_INIT,
     15 	ATTR_NORETURN,
     16 	ATTR_OFFSET,
     17 	ATTR_SYMBOL,
     18 	ATTR_TEST,
     19 	UNDERSCORE,
     20 	ABORT,
     21 	ALLOC,
     22 	APPEND,
     23 	AS,
     24 	ASSERT,
     25 	BOOL,
     26 	BREAK,
     27 	CASE,
     28 	CHAR,
     29 	CONST,
     30 	CONTINUE,
     31 	DEF,
     32 	DEFER,
     33 	DELETE,
     34 	ELSE,
     35 	ENUM,
     36 	EXPORT,
     37 	F32,
     38 	F64,
     39 	FALSE,
     40 	FN,
     41 	FOR,
     42 	FREE,
     43 	I16,
     44 	I32,
     45 	I64,
     46 	I8,
     47 	IF,
     48 	INSERT,
     49 	INT,
     50 	IS,
     51 	LEN,
     52 	LET,
     53 	MATCH,
     54 	NULL,
     55 	NULLABLE,
     56 	OFFSET,
     57 	RETURN,
     58 	RUNE,
     59 	SIZE,
     60 	STATIC,
     61 	STR,
     62 	STRUCT,
     63 	SWITCH,
     64 	TRUE,
     65 	TYPE,
     66 	U16,
     67 	U32,
     68 	U64,
     69 	U8,
     70 	UINT,
     71 	UINTPTR,
     72 	UNION,
     73 	USE,
     74 	VAARG,
     75 	VAEND,
     76 	VALIST,
     77 	VASTART,
     78 	VOID,
     79 	YIELD,
     80 	LAST_KEYWORD = YIELD,
     81 
     82 	// Operators
     83 
     84 	ARROW,
     85 	BAND,
     86 	BANDEQ,
     87 	BNOT,
     88 	BOR,
     89 	BOREQ,
     90 	BXOR,
     91 	BXOREQ,
     92 	COLON,
     93 	COMMA,
     94 	DIV,
     95 	DIVEQ,
     96 	DOT,
     97 	DOUBLE_COLON,
     98 	ELLIPSIS,
     99 	EQUAL,
    100 	GT,
    101 	GTEQ,
    102 	LAND,
    103 	LANDEQ,
    104 	LBRACE,
    105 	LBRACKET,
    106 	LEQUAL,
    107 	LESS,
    108 	LESSEQ,
    109 	LNOT,
    110 	LOR,
    111 	LOREQ,
    112 	LPAREN,
    113 	LSHIFT,
    114 	LSHIFTEQ,
    115 	LXOR,
    116 	LXOREQ,
    117 	MINUS,
    118 	MINUSEQ,
    119 	MODEQ,
    120 	MODULO,
    121 	NEQUAL,
    122 	PLUS,
    123 	PLUSEQ,
    124 	QUESTION,
    125 	RBRACE,
    126 	RBRACKET,
    127 	RPAREN,
    128 	RSHIFT,
    129 	RSHIFTEQ,
    130 	SEMICOLON,
    131 	SLICE,
    132 	TIMES,
    133 	TIMESEQ,
    134 	LAST_BTOK = TIMESEQ,
    135 
    136 	LIT_U8,
    137 	LIT_U16,
    138 	LIT_U32,
    139 	LIT_U64,
    140 	LIT_UINT,
    141 	LIT_SIZE,
    142 	LIT_I8,
    143 	LIT_I16,
    144 	LIT_I32,
    145 	LIT_I64,
    146 	LIT_INT,
    147 	LIT_ICONST,
    148 	LIT_F32,
    149 	LIT_F64,
    150 	LIT_FCONST,
    151 	LIT_RUNE,
    152 	LIT_STR,
    153 	LAST_LITERAL = LIT_STR,
    154 
    155 	NAME,
    156 	EOF,
    157 };
    158 
    159 const bmap: [_]str = [
    160 	// Keep ordered with tok
    161 	"@fini",
    162 	"@init",
    163 	"@noreturn",
    164 	"@offset",
    165 	"@symbol",
    166 	"@test",
    167 	"_",
    168 	"abort",
    169 	"alloc",
    170 	"append",
    171 	"as",
    172 	"assert",
    173 	"bool",
    174 	"break",
    175 	"case",
    176 	"char",
    177 	"const",
    178 	"continue",
    179 	"def",
    180 	"defer",
    181 	"delete",
    182 	"else",
    183 	"enum",
    184 	"export",
    185 	"f32",
    186 	"f64",
    187 	"false",
    188 	"fn",
    189 	"for",
    190 	"free",
    191 	"i16",
    192 	"i32",
    193 	"i64",
    194 	"i8",
    195 	"if",
    196 	"insert",
    197 	"int",
    198 	"is",
    199 	"len",
    200 	"let",
    201 	"match",
    202 	"null",
    203 	"nullable",
    204 	"offset",
    205 	"return",
    206 	"rune",
    207 	"size",
    208 	"static",
    209 	"str",
    210 	"struct",
    211 	"switch",
    212 	"true",
    213 	"type",
    214 	"u16",
    215 	"u32",
    216 	"u64",
    217 	"u8",
    218 	"uint",
    219 	"uintptr",
    220 	"union",
    221 	"use",
    222 	"vaarg",
    223 	"vaend",
    224 	"valist",
    225 	"vastart",
    226 	"void",
    227 	"yield",
    228 	"=>",
    229 	"&",
    230 	"&=",
    231 	"~",
    232 	"|",
    233 	"|=",
    234 	"^",
    235 	"^=",
    236 	":",
    237 	",",
    238 	"/",
    239 	"/=",
    240 	".",
    241 	"::",
    242 	"...",
    243 	"=",
    244 	">",
    245 	">=",
    246 	"&&",
    247 	"&&=",
    248 	"{",
    249 	"[",
    250 	"==",
    251 	"<",
    252 	"<=",
    253 	"!",
    254 	"||",
    255 	"||=",
    256 	"(",
    257 	"<<",
    258 	"<<=",
    259 	"^^",
    260 	"^^=",
    261 	"-",
    262 	"-=",
    263 	"%=",
    264 	"%",
    265 	"!=",
    266 	"+",
    267 	"+=",
    268 	"?",
    269 	"}",
    270 	"]",
    271 	")",
    272 	">>",
    273 	">>=",
    274 	";",
    275 	"..",
    276 	"*",
    277 	"*=",
    278 ];
    279 
    280 // The value "null".
    281 export type _null = void;
    282 
    283 // A token value, used for tokens such as '1337' (an integer).
    284 export type value = (str | rune | u64 | f64 | void);
    285 
    286 // A location within a source file.
    287 // The path is borrowed from the file name given to the lexer.
    288 export type location = struct {
    289 	path: str,
    290 	line: uint,
    291 	col: uint
    292 };
    293 
    294 // A single lexical token.
    295 export type token = (ltok, value, location);
    296 
    297 // Converts a token to its string representation.
    298 export fn tokstr(tok: token) const str = {
    299 	if (tok.0 <= ltok::LAST_BTOK) {
    300 		return bmap[tok.0: int];
    301 	};
    302 	switch (tok.0) {
    303 	case ltok::LIT_U8 =>
    304 		return "u8";
    305 	case ltok::LIT_U16 =>
    306 		return "u16";
    307 	case ltok::LIT_U32 =>
    308 		return "u32";
    309 	case ltok::LIT_U64 =>
    310 		return "u64";
    311 	case ltok::LIT_UINT =>
    312 		return "uint";
    313 	case ltok::LIT_SIZE =>
    314 		return "size";
    315 	case ltok::LIT_I8 =>
    316 		return "i8";
    317 	case ltok::LIT_I16 =>
    318 		return "i16";
    319 	case ltok::LIT_I32 =>
    320 		return "i32";
    321 	case ltok::LIT_I64 =>
    322 		return "i64";
    323 	case ltok::LIT_INT =>
    324 		return "int";
    325 	case ltok::LIT_ICONST =>
    326 		return "iconst";
    327 	case ltok::LIT_F32 =>
    328 		return "f32";
    329 	case ltok::LIT_F64 =>
    330 		return "f64";
    331 	case ltok::LIT_FCONST =>
    332 		return "fconst";
    333 	case ltok::LIT_RUNE =>
    334 		return "rune";
    335 	case ltok::LIT_STR =>
    336 		return "str";
    337 	case ltok::NAME =>
    338 		return tok.1 as str;
    339 	case ltok::EOF =>
    340 		return "EOF";
    341 	case =>
    342 		abort();
    343 	};
    344 };