123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603 |
- #include "parse/lex.h"
-
- #include <stdlib.h>
-
- static void log_token(struct l2_token *tok) {
- switch (l2_token_get_kind(tok)) {
- case L2_TOK_STRING:
- case L2_TOK_IDENT:
- case L2_TOK_ERROR:
- printf("%i:%i\t%s '%s'\n", tok->line, tok->ch,
- l2_token_get_name(tok), tok->v.str);
- break;
-
- case L2_TOK_NUMBER:
- printf("%i:%i\t%s '%g'\n", tok->line, tok->ch,
- l2_token_get_name(tok), tok->v.num);
- break;
-
- default:
- printf("%i:%i\t%s\n", tok->line, tok->ch,
- l2_token_get_name(tok));
- break;
- }
- }
-
- const char *l2_token_kind_name(enum l2_token_kind kind) {
- switch (kind) {
- case L2_TOK_OPEN_PAREN:
- return "open-paren";
- case L2_TOK_CLOSE_PAREN:
- return "close-paren";
- case L2_TOK_OPEN_BRACE:
- return "open-brace";
- case L2_TOK_CLOSE_BRACE:
- return "close-brace";
- case L2_TOK_OPEN_BRACKET:
- return "open-bracket";
- case L2_TOK_CLOSE_BRACKET:
- return "close-bracket";
- case L2_TOK_QUOT:
- return "single-quote";
- case L2_TOK_COMMA:
- return "comma";
- case L2_TOK_PERIOD:
- return "period";
- case L2_TOK_DOT_NUMBER:
- return "dot-number";
- case L2_TOK_COLON:
- return "colon";
- case L2_TOK_COLON_EQ:
- return "colon-equals";
- case L2_TOK_EQUALS:
- return "equals";
- case L2_TOK_EOL:
- return "end-of-line";
- case L2_TOK_EOF:
- return "end-of-file";
- case L2_TOK_NUMBER:
- return "number";
- case L2_TOK_STRING:
- return "string";
- case L2_TOK_IDENT:
- return "ident";
- case L2_TOK_ERROR:
- return "error";
- }
-
- return "(unknown)";
- }
-
- void l2_token_free(struct l2_token *tok) {
- enum l2_token_kind kind = l2_token_get_kind(tok);
- if (
- (kind == L2_TOK_STRING || kind == L2_TOK_IDENT) &&
- !l2_token_is_small(tok)) {
- free(tok->v.str);
- }
- }
-
- struct l2_token_value l2_token_extract_val(struct l2_token *tok) {
- struct l2_token_value v = tok->v;
- tok->v.str = NULL;
- return v;
- }
-
- void l2_lexer_init(struct l2_lexer *lexer, struct l2_io_reader *r) {
- lexer->toks[0].v.flags = L2_TOK_EOF,
- lexer->tokidx = 0;
- lexer->line = 1;
- lexer->ch = 1;
- lexer->parens = 0;
- lexer->do_log_tokens = 0;
- l2_bufio_reader_init(&lexer->reader, r);
- }
-
- static int peek_ch(struct l2_lexer *lexer) {
- int ch = l2_bufio_peek(&lexer->reader, 1);
- return ch;
- }
-
- static int read_ch(struct l2_lexer *lexer) {
- int ch = l2_bufio_get(&lexer->reader);
- lexer->ch += 1;
- if (ch == '\n') {
- lexer->ch = 1;
- lexer->line += 1;
- }
-
- return ch;
- }
-
- static int is_whitespace(int ch) {
- return ch == ' ' || ch == '\r' || ch == '\n' || ch == '\t';
- }
-
- static int is_numeric(int ch) {
- return ch >= '0' && ch <= '9';
- }
-
- static int skip_whitespace(struct l2_lexer *lexer) {
- int nl = 0;
- while (1) {
- while (is_whitespace(peek_ch(lexer))) {
- int ch = read_ch(lexer);
- if (ch == '\n') {
- nl = 1;
- }
- }
-
- if (peek_ch(lexer) == '#') {
- nl = 1;
- while (read_ch(lexer) != '\n');
- } else {
- break;
- }
- }
-
- return nl;
- }
-
- static int read_integer(struct l2_lexer *lexer, long long *num, long long *base, char **err) {
- unsigned char buffer[32]; // Should be enough
- size_t len = 0;
-
- long long b = -1;
- while (len < sizeof(buffer)) {
- int ch = peek_ch(lexer);
- if (is_numeric(ch)) {
- buffer[len++] = ch;
- } else if (ch != '\'') {
- break;
- }
-
- read_ch(lexer);
- }
-
- int ch = peek_ch(lexer);
- int abbrev_prefix = len == 1 && buffer[0] == '0';
- if (abbrev_prefix && ch == 'b') {
- b = 2;
- } else if (abbrev_prefix && ch == 'o') {
- b = 8;
- } else if (abbrev_prefix && ch == 'x') {
- b = 16;
- } else {
- // Need to parse the number as base 10 now
- long long n = 0;
- long long pow = 1;
- for (ssize_t i = len - 1; i >= 0; --i) {
- n += (buffer[i] - '0') * pow;
- pow *= 10;
- }
-
- if (ch == 'r') {
- b = n;
- } else {
- *num = n;
- *base = 10;
- return 0;
- }
- }
-
- if (b < 2) {
- *err = "Number with base lower than 2";
- return -1;
- } else if (b > 36) {
- *err = "Number with base higher than 36";
- return -1;
- }
-
- // Now that we know the base, we can read in the next part of the number
- read_ch(lexer); // Skip the base marker ('x', 'r', etc)
- len = 0;
- while (len < sizeof(buffer)) {
- int ch = peek_ch(lexer);
- if (ch == '\'') {
- read_ch(lexer);
- continue;
- }
-
- int digit;
- if (ch >= '0' && ch <= '9') {
- digit = ch - '0';
- } else if (ch >= 'a' && ch <= 'z') {
- digit = ch - 'a' + 10;
- } else if (ch >= 'A' && ch <= 'Z') {
- digit = ch - 'A' + 10;
- } else {
- break;
- }
-
- if (digit >= b) {
- *err = "Number with digit too big for the base";
- return -1;
- }
-
- buffer[len++] = digit;
- read_ch(lexer);
- }
-
- if (len < 1) {
- *err = "Number with no digits";
- }
-
- long long n = 0;
- long long pow = 1;
- for (ssize_t i = len - 1; i >= 0; --i) {
- n += buffer[i] * pow;
- pow *= b;
- }
-
- *num = n;
- *base = b;
- return 0;
- }
-
- static void read_number(struct l2_lexer *lexer, struct l2_token *tok) {
- tok->v.flags = L2_TOK_NUMBER;
-
- float sign = 1;
- if (peek_ch(lexer) == '-') {
- sign = -1;
- read_ch(lexer);
- }
-
- if (!is_numeric(peek_ch(lexer))) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "No number in number literal";
- return;
- }
-
- long long integral;
- long long base;
- char *err;
- if (read_integer(lexer, &integral, &base, &err) < 0) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = err;
- return;
- }
-
- if (peek_ch(lexer) != '.') {
- tok->v.num = (double)integral * sign;
- return;
- }
-
- read_ch(lexer); // '.'
-
- unsigned char buffer[32];
- size_t fraction_len = 0;
- while (fraction_len < sizeof(buffer)) {
- int ch = peek_ch(lexer);
- if (ch == '\'') {
- read_ch(lexer);
- continue;
- }
-
- int digit;
- if (ch >= '0' && ch <= '9') {
- digit = ch - '0';
- } else if (ch >= 'a' && ch <= 'z') {
- digit = ch - 'a' + 10;
- } else if (ch >= 'A' && ch <= 'Z') {
- digit = ch - 'A' + 10;
- } else {
- break;
- }
-
- if (digit >= base) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Number with digits too big for the base";
- return;
- }
-
- buffer[fraction_len++] = digit;
- read_ch(lexer);
- }
-
- if (fraction_len < 1) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Trailing dot in number literal";
- return;
- }
-
- long long fraction = 0;
- long long fraction_power = 1;
- for (ssize_t i = fraction_len - 1; (ssize_t)i >= 0; --i) {
- fraction += buffer[i] * fraction_power;
- fraction_power *= base;
- }
-
- double num = (double)integral + ((double)fraction / (double)fraction_power);
- tok->v.num = num * sign;
- }
-
- static void read_string(struct l2_lexer *lexer, struct l2_token *tok) {
- tok->v.flags = L2_TOK_STRING | L2_TOK_SMALL;
-
- char *dest = tok->v.strbuf;
- size_t size = sizeof(tok->v.strbuf);
- size_t idx = 0;
-
- while (1) {
- int ch = read_ch(lexer);
- if (ch == '"') {
- dest[idx] = '\0';
- return;
- } else if (ch == EOF) {
- if (!l2_token_is_small(tok)) {
- free(tok->v.str);
- }
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Unexpected EOF";
- return;
- } else if (ch == '\\') {
- int ch2 = read_ch(lexer);
- switch (ch2) {
- case 'n':
- ch = '\n';
- break;
-
- case 'r':
- ch = '\r';
- break;
-
- case 't':
- ch = '\t';
- break;
-
- case EOF:
- if (!l2_token_is_small(tok)) {
- free(tok->v.str);
- }
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Unexpected EOF";
- return;
-
- default:
- ch = ch2;
- break;
- }
- }
-
- dest[idx++] = (char)ch;
-
- // The first time we run out of space, we have to switch away from
- // the small-string optimization and malloc memory.
- if (idx + 1 >= size) {
- char *newbuf;
- if (l2_token_is_small(tok)) {
- tok->v.flags &= ~L2_TOK_SMALL;
- size = 32;
- newbuf = malloc(size);
- if (newbuf == NULL) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Allocation failure";
- return;
- }
- memcpy(newbuf, tok->v.strbuf, idx);
- } else {
- size *= 2;
- newbuf = realloc(tok->v.str, size);
- if (newbuf == NULL) {
- free(tok->v.str);
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Allocation failure";
- return;
- }
- }
-
- tok->v.str = newbuf;
- dest = newbuf;
- }
- }
- }
-
- static void read_ident(struct l2_lexer *lexer, struct l2_token *tok) {
- tok->v.flags = L2_TOK_IDENT | L2_TOK_SMALL;
-
- char *dest = tok->v.strbuf;
- size_t size = sizeof(tok->v.strbuf);
- size_t idx = 0;
-
- while (1) {
- int ch = peek_ch(lexer);
-
- if (is_whitespace(ch)) {
- dest[idx] = '\0';
- return;
- }
-
- switch (ch) {
- case '(':
- case ')':
- case '{':
- case '}':
- case '[':
- case ']':
- case '\'':
- case ',':
- case '.':
- case ':':
- case '=':
- case ';':
- case EOF:
- dest[idx] = '\0';
- return;
- }
-
- dest[idx++] = (char)read_ch(lexer);
-
- // The first time we run out of space, we have to switch away from
- // the small-string optimization and malloc memory.
- if (idx + 1 >= size) {
- char *newbuf;
- if (l2_token_is_small(tok)) {
- tok->v.flags &= ~L2_TOK_SMALL;
- size = 32;
- newbuf = malloc(size);
- if (newbuf == NULL) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Allocation failure";
- return;
- }
- memcpy(newbuf, tok->v.strbuf, idx);
- } else {
- size *= 2;
- newbuf = realloc(tok->v.str, size);
- if (newbuf == NULL) {
- free(tok->v.str);
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = "Allocation failure";
- return;
- }
- }
-
- tok->v.str = newbuf;
- dest = newbuf;
- }
- }
- }
-
- static void read_tok(struct l2_lexer *lexer, struct l2_token *tok) {
- tok->line = lexer->line;
- tok->ch = lexer->ch;
- int nl = skip_whitespace(lexer);
-
- if (nl && lexer->parens == 0) {
- tok->v.flags = L2_TOK_EOL;
- return;
- }
-
- int ch = peek_ch(lexer);
- switch (ch) {
- case '(':
- read_ch(lexer);
- tok->v.flags = L2_TOK_OPEN_PAREN;
- lexer->parens += 1;
- break;
-
- case ')':
- read_ch(lexer);
- tok->v.flags = L2_TOK_CLOSE_PAREN;
- lexer->parens -= 1;
- break;
-
- case '{':
- read_ch(lexer);
- tok->v.flags = L2_TOK_OPEN_BRACE;
- break;
-
- case '}':
- read_ch(lexer);
- tok->v.flags = L2_TOK_CLOSE_BRACE;
- break;
-
- case '[':
- read_ch(lexer);
- tok->v.flags = L2_TOK_OPEN_BRACKET;
- break;
-
- case ']':
- read_ch(lexer);
- tok->v.flags = L2_TOK_CLOSE_BRACKET;
- break;
-
- case ';':
- tok->v.flags = L2_TOK_EOL;
- do {
- read_ch(lexer);
- skip_whitespace(lexer);
- } while (peek_ch(lexer) == ';');
- break;
-
- case '\'':
- read_ch(lexer);
- tok->v.flags = L2_TOK_QUOT;
- break;
-
- case ',':
- read_ch(lexer);
- tok->v.flags = L2_TOK_COMMA;
- break;
-
- case '.':
- read_ch(lexer);
- if (is_numeric(peek_ch(lexer))) {
- tok->v.flags = L2_TOK_DOT_NUMBER;
- long long num, base;
- char *err;
- if (read_integer(lexer, &num, &base, &err) < 0) {
- tok->v.flags = L2_TOK_ERROR;
- tok->v.str = err;
- } else {
- tok->v.integer = (int)num;
- }
- } else {
- tok->v.flags = L2_TOK_PERIOD;
- }
- break;
-
- case ':':
- read_ch(lexer);
- ch = peek_ch(lexer);
- switch (ch) {
- case '=':
- read_ch(lexer);
- tok->v.flags = L2_TOK_COLON_EQ;
- break;
-
- default:
- tok->v.flags = L2_TOK_COLON;
- break;
- }
- break;
-
- case '=':
- read_ch(lexer);
- tok->v.flags = L2_TOK_EQUALS;
- break;
-
- case EOF:
- tok->v.flags = L2_TOK_EOF;
- break;
-
- case '"':
- read_ch(lexer);
- read_string(lexer, tok);
- break;
-
- default:
- if (is_numeric(ch) || ch == '-') {
- read_number(lexer, tok);
- break;
- }
-
- read_ident(lexer, tok);
- }
- }
-
- struct l2_token *l2_lexer_peek(struct l2_lexer *lexer, int count) {
- int offset = count - 1;
-
- while (offset >= lexer->tokidx) {
- read_tok(lexer, &lexer->toks[lexer->tokidx++]);
- if (lexer->do_log_tokens) {
- log_token(&lexer->toks[lexer->tokidx - 1]);
- }
- }
-
- return &lexer->toks[offset];
- }
-
- void l2_lexer_consume(struct l2_lexer *lexer) {
- l2_token_free(&lexer->toks[0]);
- lexer->tokidx -= 1;
- memmove(lexer->toks, lexer->toks + 1, lexer->tokidx * sizeof(*lexer->toks));
- }
-
- void l2_lexer_skip_opt(struct l2_lexer *lexer, enum l2_token_kind kind) {
- if (l2_token_get_kind(l2_lexer_peek(lexer, 1)) == kind) {
- l2_lexer_consume(lexer);
- }
- }
|