l3-ENSL/c-repl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add code position to tokens

Browse Source
This commit is contained in:
ala89 2023-11-15 14:31:11 +01:00
parent 87bcd0d064
commit cb3444fcf3
5 changed files with 105 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -14,7 +14,7 @@ TESTS_OBJ = $(TESTS_SRC:$(TEST_SRCDIR)/%.cpp=$(BUILDDIR)/test-%)
LD_CXXFLAGS = LD_CXXFLAGS =
# Compilation flag # Compilation flag
CXXFLAGS = -Wall -Wextra -g -O3 -std=c++2a CXXFLAGS = -Wall -Wextra -g -O0 -std=c++2a
# Remove warnings about unused variables, functions, ... # Remove warnings about unused variables, functions, ...
# -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable # -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable
# Compile with debug # Compile with debug

187
src/tokenize.cpp
View File

@ -1,6 +1,7 @@
#include <regex> #include <regex>
#include <vector> #include <vector>
#include <iostream> #include <iostream>
#include <string>
#include "include/tokenize.h" #include "include/tokenize.h"
using namespace std; using namespace std;
@ -63,95 +64,107 @@ void _debug_print_tokens(vector<Token> tokens) {
cout << endl; cout << endl;
} }
vector<Token> tokenize(string str) { vector<Token> tokenize(vector<string> input) {
vector<Token> tokens; vector<Token> tokens;
while (str.size() > 0) { for (int i = 0; i < int(input.size()); i++) {
smatch m; string line = input[i];
if (regex_search(str, m, NUMBER_REGEX, regex_constants::match_continuous)) { int j = 0;
Token token = {
.type = TokenType::Int, CodePosition pos = { .line = i, .column = j };
.data = stoi(m.str())
}; while (j < int(line.length())) {
tokens.emplace_back(token); string str = line.substr(j, string::npos);
str.erase(0, m.str().length()); smatch m;
}
else if (regex_search(str, m, TYPE_INT_REGEX, regex_constants::match_continuous)) { if (regex_search(str, m, NUMBER_REGEX, regex_constants::match_continuous)) {
Token token = { Token token = {
.type = TokenType::Type, .type = TokenType::Int,
.data = Type::Int .data = stoi(m.str()),
}; .pos = pos
tokens.emplace_back(token); };
str.erase(0, m.str().length()); tokens.emplace_back(token);
} j += m.str().length();
else if (regex_search(str, m, IDENTIFIER_REGEX, regex_constants::match_continuous)) { }
Token token = { else if (regex_search(str, m, TYPE_INT_REGEX, regex_constants::match_continuous)) {
.type = TokenType::Identifier, Token token = {
.data = m.str() .type = TokenType::Type,
}; .data = Type::Int,
tokens.emplace_back(token); .pos = pos
str.erase(0, m.str().length()); };
} tokens.emplace_back(token);
else if (str.size() >= 2 && str[0] == '+' && str[1] == '+') { j += m.str().length();
Token token = { .type = TokenType::DoublePlus }; }
tokens.emplace_back(token); else if (regex_search(str, m, IDENTIFIER_REGEX, regex_constants::match_continuous)) {
str.erase(0, 2); Token token = {
} .type = TokenType::Identifier,
else if (str.size() >= 2 && str[0] == '-' && str[1] == '-') { .data = m.str(),
Token token = { .type = TokenType::DoubleMinus }; .pos = pos
tokens.emplace_back(token); };
str.erase(0, 2); tokens.emplace_back(token);
} j += m.str().length();
else if (str[0] == '+') { }
Token token = { .type = TokenType::Plus }; else if (str.length() >= 2 && str[0] == '+' && str[1] == '+') {
tokens.emplace_back(token); Token token = { .type = TokenType::DoublePlus, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 2;
else if (str[0] == '-') { }
Token token = { .type = TokenType::Minus }; else if (str.length() >= 2 && str[0] == '-' && str[1] == '-') {
tokens.emplace_back(token); Token token = { .type = TokenType::DoubleMinus, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 2;
else if (str[0] == '*') { }
Token token = { .type = TokenType::Star }; else if (str[0] == '+') {
tokens.emplace_back(token); Token token = { .type = TokenType::Plus, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == '/') { }
Token token = { .type = TokenType::Slash }; else if (str[0] == '-') {
tokens.emplace_back(token); Token token = { .type = TokenType::Minus, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == '%') { }
Token token = { .type = TokenType::Percent }; else if (str[0] == '*') {
tokens.emplace_back(token); Token token = { .type = TokenType::Star, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == '=') { }
Token token = { .type = TokenType::Equal }; else if (str[0] == '/') {
tokens.emplace_back(token); Token token = { .type = TokenType::Slash, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == ';') { }
Token token = { .type = TokenType::Semicolon }; else if (str[0] == '%') {
tokens.emplace_back(token); Token token = { .type = TokenType::Percent, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == '(') { }
Token token = { .type = TokenType::LParenthese }; else if (str[0] == '=') {
tokens.emplace_back(token); Token token = { .type = TokenType::Equal, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (str[0] == ')') { }
Token token = { .type = TokenType::RParenthese }; else if (str[0] == ';') {
tokens.emplace_back(token); Token token = { .type = TokenType::Semicolon, .pos = pos };
str.erase(0, 1); tokens.emplace_back(token);
} j += 1;
else if (isspace(str[0]) || str[0] == '\0') { }
str.erase(0, 1); else if (str[0] == '(') {
} Token token = { .type = TokenType::LParenthese, .pos = pos };
else { tokens.emplace_back(token);
throw TokenError("Unknown token {}"); j += 1;
}
else if (str[0] == ')') {
Token token = { .type = TokenType::RParenthese, .pos = pos };
tokens.emplace_back(token);
j += 1;
}
else if (isspace(str[0]) || str[0] == '\0') {
j += 1;
}
else {
throw TokenError("Unknown token {}");
}
} }
} }

2
test/expr_arithmetiques.cpp
View File

@ -5,7 +5,7 @@
#include "../src/include/interpreter.h" #include "../src/include/interpreter.h"
int execute(string s) { int execute(string s) {
vector<Token> tokens = tokenize(s); vector<Token> tokens = tokenize({ s });
Node ast = parse(tokens); Node ast = parse(tokens);
EvalResult res = eval(ast); EvalResult res = eval(ast);

4
test/tokenize.cpp
View File

@ -13,7 +13,7 @@ int main() {
{ {
string input = "int a = x+++7;"; string input = "int a = x+++7;";
vector<Token> tokens = tokenize(input); vector<Token> tokens = tokenize({ input });
_TEST_ASSERT( _TEST_ASSERT(
_TEST_NO_EXCEPTION(tokens.size() == 8), _TEST_NO_EXCEPTION(tokens.size() == 8),
@ -97,7 +97,7 @@ int main() {
string input = "int a = 10 @;"; string input = "int a = 10 @;";
_TEST_ASSERT( _TEST_ASSERT(
_TEST_IS_EXCEPTION(tokenize(input), TokenError), _TEST_IS_EXCEPTION(tokenize({ input }), TokenError),
"Token invalide", "Token invalide",
true true
) )

2
test/variables.cpp
View File

@ -5,7 +5,7 @@
#include "../src/include/interpreter.h" #include "../src/include/interpreter.h"
int execute(string s) { int execute(string s) {
vector<Token> tokens = tokenize(s); vector<Token> tokens = tokenize({ s });
Node ast = parse(tokens); Node ast = parse(tokens);
EvalResult res = eval(ast); EvalResult res = eval(ast);