Add tokenization for conditionnal statements

This commit is contained in:
ala89 2023-11-15 17:15:18 +01:00
parent 5e304a70aa
commit 18f69eef08
2 changed files with 54 additions and 14 deletions

View File

@ -7,7 +7,7 @@
#include <stdexcept> #include <stdexcept>
using namespace std; using namespace std;
enum class TokenType { Type, Identifier, Int, Plus, Minus, DoublePlus, DoubleMinus, Star, Slash, Percent, Equal, Semicolon, LParenthese, RParenthese, LCurlyBracket, RCurlyBracket, If, Else }; enum class TokenType { Type, Identifier, Int, Plus, Minus, DoublePlus, DoubleMinus, DoubleEqual, Star, Slash, Percent, Equal, Semicolon, LParenthese, RParenthese, LCurlyBracket, RCurlyBracket, If, Else };
enum class Type { Int }; enum class Type { Int };
using TokenData = variant<int, string, Type>; using TokenData = variant<int, string, Type>;

View File

@ -32,6 +32,9 @@ void _debug_print_token(Token token) {
case TokenType::DoubleMinus: case TokenType::DoubleMinus:
cout << "--"; cout << "--";
break; break;
case TokenType::DoubleEqual:
cout << "==";
break;
case TokenType::Star: case TokenType::Star:
cout << "*"; cout << "*";
break; break;
@ -53,6 +56,18 @@ void _debug_print_token(Token token) {
case TokenType::RParenthese: case TokenType::RParenthese:
cout << ")"; cout << ")";
break; break;
case TokenType::LCurlyBracket:
cout << "{";
break;
case TokenType::RCurlyBracket:
cout << "}";
break;
case TokenType::If:
cout << "If";
break;
case TokenType::Else:
cout << "Else";
break;
} }
} }
@ -103,66 +118,91 @@ vector<Token> tokenize(vector<string> input, int initial_line) {
tokens.emplace_back(token); tokens.emplace_back(token);
j += m.str().length(); j += m.str().length();
} }
else if (str.length() >= 2 && str[0] == '+' && str[1] == '+') { else if (str.starts_with("++")) {
Token token = { .type = TokenType::DoublePlus, .pos = pos }; Token token = { .type = TokenType::DoublePlus, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 2; j += 2;
} }
else if (str.length() >= 2 && str[0] == '-' && str[1] == '-') { else if (str.starts_with("--")) {
Token token = { .type = TokenType::DoubleMinus, .pos = pos }; Token token = { .type = TokenType::DoubleMinus, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 2; j += 2;
} }
else if (str[0] == '+') { else if (str.starts_with("==")) {
Token token = { .type = TokenType::DoubleEqual, .pos = pos };
tokens.emplace_back(token);
j += 2;
}
else if (str.starts_with("+")) {
Token token = { .type = TokenType::Plus, .pos = pos }; Token token = { .type = TokenType::Plus, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '-') { else if (str.starts_with("-")) {
Token token = { .type = TokenType::Minus, .pos = pos }; Token token = { .type = TokenType::Minus, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '*') { else if (str.starts_with("*")) {
Token token = { .type = TokenType::Star, .pos = pos }; Token token = { .type = TokenType::Star, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '/') { else if (str.starts_with("/")) {
Token token = { .type = TokenType::Slash, .pos = pos }; Token token = { .type = TokenType::Slash, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '%') { else if (str.starts_with("%")) {
Token token = { .type = TokenType::Percent, .pos = pos }; Token token = { .type = TokenType::Percent, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '=') { else if (str.starts_with("=")) {
Token token = { .type = TokenType::Equal, .pos = pos }; Token token = { .type = TokenType::Equal, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == ';') { else if (str.starts_with(";")) {
Token token = { .type = TokenType::Semicolon, .pos = pos }; Token token = { .type = TokenType::Semicolon, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == '(') { else if (str.starts_with("(")) {
Token token = { .type = TokenType::LParenthese, .pos = pos }; Token token = { .type = TokenType::LParenthese, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (str[0] == ')') { else if (str.starts_with(")")) {
Token token = { .type = TokenType::RParenthese, .pos = pos }; Token token = { .type = TokenType::RParenthese, .pos = pos };
tokens.emplace_back(token); tokens.emplace_back(token);
j += 1; j += 1;
} }
else if (isspace(str[0]) || str[0] == '\0') { else if (str.starts_with("{")) {
Token token = { .type = TokenType::LCurlyBracket, .pos = pos };
tokens.emplace_back(token);
j += 1;
}
else if (str.starts_with("}")) {
Token token = { .type = TokenType::RCurlyBracket, .pos = pos };
tokens.emplace_back(token);
j += 1;
}
else if (str.starts_with("if")) {
Token token = { .type = TokenType::If, .pos = pos };
tokens.emplace_back(token);
j += 2;
}
else if (str.starts_with("else")) {
Token token = { .type = TokenType::Else, .pos = pos };
tokens.emplace_back(token);
j += 4;
}
else if (isspace(str[0])) {
j += 1; j += 1;
} }
else { else {
throw TokenError("Unknown token {}", pos); throw TokenError("Unknown token", pos);
} }
} }
} }