1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
#include "lexer/lexer.h"
Lexer::Lexer(const std::string& src) : src(src), position(0), line(1), column(1) {}
std::vector<Token> Lexer::tokenise() {
while (position < src.size()) {
if (std::isdigit(src[position])) {
std::string digit;
int original_column = column;
while (std::isdigit(src[position])) {
digit.push_back(src[position]);
advance();
}
tokens.push_back(Token(TokenType::NUMBER, digit, line, original_column));
} else if (std::isalpha(src[position])) {
if (src.substr(position, 3) == "int") {
tokens.push_back(Token(TokenType::INT, "int", line, column));
advance();
advance();
advance();
} else {
std::string identifier;
int original_column = column;
while (std::isalnum(src[position])) {
identifier.push_back(src[position]);
advance();
}
tokens.push_back(Token(TokenType::IDENTIFIER, identifier, line, original_column));
}
} else if (src[position] == '+') {
tokens.push_back(Token(TokenType::PLUS, std::string(1, src[position]), line, column));
advance();
} else if (src[position] == '-') {
tokens.push_back(Token(TokenType::MINUS, std::string(1, src[position]), line, column));
advance();
} else if (src[position] == '*') {
tokens.push_back(Token(TokenType::MULTIPLY, std::string(1, src[position]), line, column));
advance();
} else if (src[position] == '/') {
tokens.push_back(Token(TokenType::DIVIDE, std::string(1, src[position]), line, column));
advance();
} else if (src[position] == '=') {
tokens.push_back(Token(TokenType::ASSIGN, std::string(1, src[position]), line, column));
advance();
} else if (src[position] == ';') {
tokens.push_back(Token(TokenType::SEMICOLON, std::string(1, src[position]), line, column));
advance();
} else {
tokens.push_back(Token(TokenType::INVALID, std::string(1, src[position]), line, column));
advance();
}
skipComment();
skipWhitespace();
}
tokens.push_back(Token(TokenType::END_OF_FILE, std::string(), line, column));
return tokens;
}
char Lexer::advance() {
column++;
return src[position++];
}
void Lexer::skipWhitespace() {
while (src[position] == ' ' || src[position] == '\n') {
if (src[position] == ' ') {
position++;
column++;
} else if (src[position] == '\n') {
position++;
column = 1;
line++;
}
}
}
void Lexer::skipComment() {
if (src[position] == '/' && src[++position] == '/') {
while (src[position] != '\n') {
position++;
column++;
}
position++;
line += 1;
column = 1;
}
}
|
