c-repl/src/tokenize.cpp

#include <regex>
#include <vector>
#include <iostream>
#include <string>
#include "include/tokenize.h"
using namespace std;

regex NUMBER_REGEX ("\\d+(\\.\\d+)?");
regex TYPE_INT_REGEX ("int\\s");
regex IDENTIFIER_REGEX ("[A-Za-z_]\\w*");

void _debug_print_token(Token token) {
    switch (token.type) {
        case TokenType::Type:
            cout << "Type(INT)";
        break;
        case TokenType::Int:
            cout << "Number(" << get<int>(token.data) << ")";
        break;
        case TokenType::Identifier:
            cout << "Identifier(" << get<string>(token.data) << ")";
        break;
        case TokenType::Plus:
            cout << "+";
        break;
        case TokenType::Minus:
            cout << "-";
        break;
        case TokenType::DoublePlus:
            cout << "++";
        break;
        case TokenType::DoubleMinus:
            cout << "--";
        break;
        case TokenType::DoubleEqual:
            cout << "==";
        break;
        case TokenType::Star:
            cout << "*";
        break;
        case TokenType::Slash:
            cout << "/";
        break;
        case TokenType::Percent:
            cout << "%";
        break;
        case TokenType::Equal:
            cout << "=";
        break;
        case TokenType::Semicolon:
            cout << ";";
        break;
        case TokenType::LParenthese:
            cout << "(";
        break;
        case TokenType::RParenthese:
            cout << ")";
        break;
        case TokenType::LCurlyBracket:
            cout << "{";
        break;
        case TokenType::RCurlyBracket:
            cout << "}";
        break;
        case TokenType::If:
            cout << "If";
        break;
        case TokenType::Else:
            cout << "Else";
        break;
    }
}

void _debug_print_tokens(vector<Token> tokens) {
    for (Token token : tokens) {
        _debug_print_token(token);
        cout << " ";
    }
    cout << endl;
}

vector<Token> tokenize(vector<string> input, int initial_line) {
    vector<Token> tokens;

    for (int i = initial_line; i < int(input.size()); i++) {
        string line = input[i];
        int j = 0;
        
        while (j < int(line.length())) {
            CodePosition pos = { .line = i, .column = j };
            string str = line.substr(j, string::npos);
            smatch m;

            if (regex_search(str, m, NUMBER_REGEX, regex_constants::match_continuous)) {
                Token token = {
                    .type = TokenType::Int,
                    .data = stoi(m.str()),
                    .pos = pos
                };
                tokens.emplace_back(token);
                j += m.str().length();
            }
            else if (regex_search(str, m, TYPE_INT_REGEX, regex_constants::match_continuous)) {
                Token token = {
                    .type = TokenType::Type,
                    .data = Type::Int,
                    .pos = pos
                };
                tokens.emplace_back(token);
                j += m.str().length();
            }
            else if (regex_search(str, m, IDENTIFIER_REGEX, regex_constants::match_continuous)) {
                Token token = {
                    .type = TokenType::Identifier,
                    .data = m.str(),
                    .pos = pos
                };
                tokens.emplace_back(token);
                j += m.str().length();
            }
            else if (str.starts_with("++")) {
                Token token = { .type = TokenType::DoublePlus, .pos = pos };
                tokens.emplace_back(token);
                j += 2;
            }
            else if (str.starts_with("--")) {
                Token token = { .type = TokenType::DoubleMinus, .pos = pos };
                tokens.emplace_back(token);
                j += 2;
            }
            else if (str.starts_with("==")) {
                Token token = { .type = TokenType::DoubleEqual, .pos = pos };
                tokens.emplace_back(token);
                j += 2;
            }
            else if (str.starts_with("+")) {
                Token token = { .type = TokenType::Plus, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("-")) {
                Token token = { .type = TokenType::Minus, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("*")) {
                Token token = { .type = TokenType::Star, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("/")) {
                Token token = { .type = TokenType::Slash, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("%")) {
                Token token = { .type = TokenType::Percent, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("=")) {
                Token token = { .type = TokenType::Equal, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with(";")) {
                Token token = { .type = TokenType::Semicolon, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("(")) {
                Token token = { .type = TokenType::LParenthese, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with(")")) {
                Token token = { .type = TokenType::RParenthese, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("{")) {
                Token token = { .type = TokenType::LCurlyBracket, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("}")) {
                Token token = { .type = TokenType::RCurlyBracket, .pos = pos };
                tokens.emplace_back(token);
                j += 1;
            }
            else if (str.starts_with("if")) {
                Token token = { .type = TokenType::If, .pos = pos };
                tokens.emplace_back(token);
                j += 2;
            }
            else if (str.starts_with("else")) {
                Token token = { .type = TokenType::Else, .pos = pos };
                tokens.emplace_back(token);
                j += 4;
            }
            else if (isspace(str[0])) {
                j += 1;
            }
            else {
                throw TokenError("Unknown token", pos);
            }
        }
    }

    return tokens;
}