30 lines
524 B
C++
30 lines
524 B
C++
#ifndef TOKENIZE_H
|
|
#define TOKENIZE_H
|
|
|
|
#include <vector>
|
|
#include <string>
|
|
#include <stdexcept>
|
|
#include "types.h"
|
|
using namespace std;
|
|
|
|
/*
|
|
Parses a string into a vector of tokens
|
|
*/
|
|
vector<Token> tokenize(vector<string> str, int initial_line=0);
|
|
|
|
/*
|
|
Format and print a Token
|
|
*/
|
|
void _debug_print_token(Token token);
|
|
|
|
/*
|
|
Returns the name of a TokenType
|
|
*/
|
|
string _debug_get_token_type_name(TokenType type);
|
|
|
|
/*
|
|
Formats a list of tokens and prints it
|
|
*/
|
|
void _debug_print_tokens(vector<Token> tokens);
|
|
|
|
#endif |