1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
#[derive(Debug, PartialEq)]
enum Token {
Number(f64),
Operator(char),
Function(String),
Variable(String),
RightParen,
LeftParen
}
#[derive(Debug, PartialEq)]
enum TokenizerError {
TooManyDots(String),
ConstantNotFound(String),
BadCharacter(char)
}
fn str_to_const(s: &str) -> Option<f64> {
match s {
"PI" => Some(std::f64::consts::PI),
"E" => Some(std::f64::consts::E),
_ => None
}
}
fn tokenize(input: &str) -> Result<Vec<Token>, TokenizerError> {
let mut tokens = Vec::new();
let mut chars = input.chars().peekable();
while let Some(&ch) = chars.peek() {
match ch {
'0'..='9' => {
let mut num = String::new();
let mut has_dot = false;
while let Some(&c) = chars.peek() {
if c.is_ascii_digit() || (c == '.' && !has_dot) {
num.push(c);
chars.next();
has_dot |= c == '.';
} else if c == '.' && has_dot {
return Err(TokenizerError::TooManyDots(num));
} else {
break;
}
}
tokens.push(Token::Number(num.parse().unwrap()));
}
'+' | '-' | '*' | '/' | '%' | '^' => {
chars.next();
tokens.push(Token::Operator(ch));
}
'a'..='z' | '_' => {
let mut pt: Option<Token> = None;
let mut s = String::new();
while let Some(&c) = chars.peek() {
match c {
'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
s.push(c);
chars.next();
}
'(' => {
pt = Some(Token::Function(s.clone()));
break;
}
_ => {
break;
}
}
}
tokens.push(match pt {
Some(token) => token,
None => Token::Variable(s.clone())
});
}
'A'..='Z' => {
let mut s = String::new();
while let Some(&c @ ('A'..='Z' | '_')) = chars.peek() {
s.push(c);
chars.next();
}
let n = str_to_const(&s).ok_or(TokenizerError::ConstantNotFound(s))?;
tokens.push(Token::Number(n));
}
c if c.is_whitespace() || c == ',' => {
while let Some(&c) = chars.peek() {
if c.is_whitespace() {
chars.next();
} else {
break;
}
}
}
'(' => {
chars.next();
tokens.push(Token::RightParen);
}
')' => {
chars.next();
tokens.push(Token::LeftParen);
}
_ => {
return Err(TokenizerError::BadCharacter(ch));
}
}
}
Ok(tokens)
}
fn main() {
let input = "x + 4 * (3 - sin((y+PI)) / 2.123)^2";
println!("{}", input);
let tokens = tokenize(input).unwrap();
println!("{:#?}", tokens);
}
|
