commit 31c7f1381d12aba90f690945d4c4f1cd87782e88
parent f3d7aa01e280d8cdeac6106fd808b9cffd6f57ff
Author: Drew DeVault <sir@cmpwn.com>
Date: Sun, 22 Nov 2020 09:10:28 -0500
lex: add unlex
Diffstat:
2 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/include/lex.h b/include/lex.h
@@ -138,11 +138,13 @@ struct lexer {
char *buf;
size_t bufsz, buflen;
uint32_t c[2];
+ struct token un;
};
void lex_init(struct lexer *lexer, FILE *f);
void lex_finish(struct lexer *lexer);
enum lexical_token lex(struct lexer *lexer, struct token *out);
+void unlex(struct lexer *lexer, struct token *in);
void token_finish(struct token *tok);
const char *token_str(const struct token *tok);
diff --git a/src/lex.c b/src/lex.c
@@ -117,6 +117,7 @@ lex_init(struct lexer *lexer, FILE *f)
lexer->in = f;
lexer->bufsz = 256;
lexer->buf = calloc(1, lexer->bufsz);
+ lexer->un.token = T_ERROR;
}
void
@@ -711,6 +712,12 @@ lex2(struct lexer *lexer, struct token *out, uint32_t c)
enum lexical_token
lex(struct lexer *lexer, struct token *out)
{
+ if (lexer->un.token != T_ERROR) {
+ *out = lexer->un;
+ lexer->un.token = T_ERROR;
+ return out->token;
+ }
+
uint32_t c = wgetc(lexer);
if (c == UTF8_INVALID) {
out->token = T_EOF;
@@ -836,3 +843,10 @@ token_str(const struct token *tok)
return lexical_token_str(tok->token);
}
}
+
+void
+unlex(struct lexer *lexer, struct token *in)
+{
+ assert(lexer->un.token == T_ERROR && "Only one unlex is supported");
+ lexer->un = *in;
+}