2023-10-27 14:46:32 +02:00
|
|
|
#ifndef TOKENIZE_H
|
|
|
|
#define TOKENIZE_H
|
|
|
|
|
2023-10-27 14:37:03 +02:00
|
|
|
#include <vector>
|
2023-10-27 17:53:58 +02:00
|
|
|
#include <string>
|
2023-11-15 11:59:38 +01:00
|
|
|
#include <stdexcept>
|
2023-11-22 13:52:16 +01:00
|
|
|
#include "types.h"
|
2023-10-27 14:37:03 +02:00
|
|
|
using namespace std;
|
|
|
|
|
2023-11-10 17:35:33 +01:00
|
|
|
/*
|
|
|
|
Parses a string into a vector of tokens
|
|
|
|
*/
|
2023-11-15 16:07:50 +01:00
|
|
|
vector<Token> tokenize(vector<string> str, int initial_line=0);
|
2023-11-10 17:35:33 +01:00
|
|
|
|
2023-11-10 19:04:24 +01:00
|
|
|
/*
|
|
|
|
Format and print a Token
|
|
|
|
*/
|
|
|
|
void _debug_print_token(Token token);
|
|
|
|
|
2023-11-24 10:21:58 +01:00
|
|
|
/*
|
|
|
|
Returns the name of a TokenType
|
|
|
|
*/
|
2023-12-08 15:59:45 +01:00
|
|
|
string _debug_get_token_type_name(TokenType type);
|
2023-11-24 10:21:58 +01:00
|
|
|
|
2023-11-10 17:35:33 +01:00
|
|
|
/*
|
|
|
|
Formats a list of tokens and prints it
|
|
|
|
*/
|
|
|
|
void _debug_print_tokens(vector<Token> tokens);
|
|
|
|
|
2023-10-27 14:46:32 +02:00
|
|
|
#endif
|