polyspec_crudui_validator/expr/
lexer.rs

1//! CRUDUI expression lexer (expressions.md §1, JS/PHP `Lexer` parity).
2//!
3//! Turns a condition string into a `Vec<Token>` ending in EOF. WHITESPACE is
4//! consumed but never emitted. The token order, multi-char operator precedence,
5//! multi-dot handling, string escapes, and number rules mirror validator-ts
6//! `Lexer` exactly, because the shared fixture's `tokens` array is byte-compared
7//! across 4 languages.
8//!
9//! NO regex-split / no string-split evaluation anywhere (GRAMMAR §10): a single
10//! forward byte scan. Identifiers are ASCII (`[A-Za-z_][A-Za-z0-9_]*`), so
11//! byte-wise scanning is exact.
12
13use super::error::ParseError;
14use super::token::{Literal, Token, TokenType};
15
16/// Forward-scanning tokenizer over a single expression string.
17pub struct Lexer {
18    input: Vec<u8>,
19    position: usize,
20}
21
22impl Lexer {
23    /// Build a lexer over the given expression string.
24    pub fn new(input: &str) -> Self {
25        Lexer {
26            input: input.as_bytes().to_vec(),
27            position: 0,
28        }
29    }
30
31    /// Tokenize the whole input. WHITESPACE tokens are dropped; an EOF token is
32    /// always appended last.
33    pub fn tokenize(&mut self) -> Result<Vec<Token>, ParseError> {
34        let mut tokens = Vec::new();
35
36        while !self.is_at_end() {
37            let token = self.next_token()?;
38            if token.token_type != TokenType::Whitespace {
39                tokens.push(token);
40            }
41        }
42
43        tokens.push(Token::new(TokenType::Eof, "", Literal::None));
44        Ok(tokens)
45    }
46
47    fn next_token(&mut self) -> Result<Token, ParseError> {
48        let start = self.position;
49
50        // Whitespace
51        if self.match_whitespace() {
52            let raw = self.slice(start, self.position);
53            return Ok(Token::new(TokenType::Whitespace, raw, Literal::None));
54        }
55
56        // Multi-character operators first (order matters: longest / specific first)
57        if self.match_string("not in") || self.match_string("not  in") {
58            return Ok(Token::new(TokenType::NotIn, "not in", Literal::None));
59        }
60        if self.match_string("&&") {
61            return Ok(Token::new(TokenType::And, "&&", Literal::None));
62        }
63        if self.match_string("||") {
64            return Ok(Token::new(TokenType::Or, "||", Literal::None));
65        }
66        if self.match_string("==") {
67            return Ok(Token::new(TokenType::Eq, "==", Literal::None));
68        }
69        if self.match_string("!=") {
70            return Ok(Token::new(TokenType::Ne, "!=", Literal::None));
71        }
72        if self.match_string(">=") {
73            return Ok(Token::new(TokenType::Ge, ">=", Literal::None));
74        }
75        if self.match_string("<=") {
76            return Ok(Token::new(TokenType::Le, "<=", Literal::None));
77        }
78        if self.match_string(">") {
79            return Ok(Token::new(TokenType::Gt, ">", Literal::None));
80        }
81        if self.match_string("<") {
82            return Ok(Token::new(TokenType::Lt, "<", Literal::None));
83        }
84
85        // NOT operator (single !), only when not part of !=
86        if self.peek() == b'!' && self.peek_next() != b'=' {
87            self.advance();
88            return Ok(Token::new(TokenType::Not, "!", Literal::None));
89        }
90
91        // Multi-dot (.., ..., etc.) vs single dot
92        if self.peek() == b'.' {
93            let mut dot_count: i64 = 0;
94            let dot_start = self.position;
95            while self.peek() == b'.' {
96                dot_count += 1;
97                self.advance();
98            }
99
100            if dot_count > 1 {
101                let raw = self.slice(dot_start, self.position);
102                return Ok(Token::new(
103                    TokenType::DotDot,
104                    raw,
105                    Literal::DotCount(dot_count),
106                ));
107            }
108            return Ok(Token::new(TokenType::Dot, ".", Literal::None));
109        }
110
111        if self.match_string("*") {
112            return Ok(Token::new(TokenType::Asterisk, "*", Literal::None));
113        }
114        if self.match_string("(") {
115            return Ok(Token::new(TokenType::LParen, "(", Literal::None));
116        }
117        if self.match_string(")") {
118            return Ok(Token::new(TokenType::RParen, ")", Literal::None));
119        }
120        if self.match_string("[") {
121            return Ok(Token::new(TokenType::LBracket, "[", Literal::None));
122        }
123        if self.match_string("]") {
124            return Ok(Token::new(TokenType::RBracket, "]", Literal::None));
125        }
126        if self.match_string(",") {
127            return Ok(Token::new(TokenType::Comma, ",", Literal::None));
128        }
129        if self.match_string("?") {
130            return Ok(Token::new(TokenType::Question, "?", Literal::None));
131        }
132        if self.match_string(":") {
133            return Ok(Token::new(TokenType::Colon, ":", Literal::None));
134        }
135
136        // String literal
137        let ch = self.peek();
138        if ch == b'\'' || ch == b'"' {
139            return self.read_string();
140        }
141
142        // Number: leading digit, or leading '-' directly followed by a digit
143        if is_digit(ch) || (ch == b'-' && is_digit(self.peek_next())) {
144            return self.read_number();
145        }
146
147        // Keyword / identifier
148        if is_alpha(ch) {
149            return Ok(self.read_identifier());
150        }
151
152        // Unknown single character
153        let raw = self.slice(start, start + 1);
154        self.advance();
155        Ok(Token::new(TokenType::Invalid, raw, Literal::None))
156    }
157
158    fn read_string(&mut self) -> Result<Token, ParseError> {
159        let quote = self.advance();
160        let mut value = String::new();
161
162        while !self.is_at_end() && self.peek() != quote {
163            if self.peek() == b'\\' {
164                self.advance();
165                if !self.is_at_end() {
166                    let escaped = self.advance();
167                    match escaped {
168                        b'n' => value.push('\n'),
169                        b't' => value.push('\t'),
170                        b'r' => value.push('\r'),
171                        b'\\' => value.push('\\'),
172                        b'\'' => value.push('\''),
173                        b'"' => value.push('"'),
174                        other => value.push(other as char),
175                    }
176                }
177            } else {
178                value.push(self.advance() as char);
179            }
180        }
181
182        if self.is_at_end() {
183            return Err(ParseError::new("Unterminated string literal"));
184        }
185
186        self.advance(); // closing quote
187
188        // JS stores value = quote + content + quote (raw), literal = decoded.
189        let q = quote as char;
190        let raw = format!("{}{}{}", q, value, q);
191        Ok(Token::new(TokenType::String, raw, Literal::Str(value)))
192    }
193
194    fn read_number(&mut self) -> Result<Token, ParseError> {
195        let start = self.position;
196
197        if self.peek() == b'-' {
198            self.advance();
199        }
200
201        while is_digit(self.peek()) {
202            self.advance();
203        }
204
205        let mut is_float = false;
206        if self.peek() == b'.' && is_digit(self.peek_next()) {
207            is_float = true;
208            self.advance(); // consume '.'
209            while is_digit(self.peek()) {
210                self.advance();
211            }
212        }
213
214        // Scientific notation (JS parity).
215        if self.peek() == b'e' || self.peek() == b'E' {
216            is_float = true;
217            self.advance();
218            if self.peek() == b'+' || self.peek() == b'-' {
219                self.advance();
220            }
221            while is_digit(self.peek()) {
222                self.advance();
223            }
224        }
225
226        let raw = self.slice(start, self.position);
227
228        // Keep integers as integers so fixture integer literals stay integers;
229        // fractional/exponent forms become floats.
230        let literal = if is_float {
231            Literal::Float(parse_float(&raw)?)
232        } else {
233            match raw.parse::<i64>() {
234                Ok(n) => Literal::Int(n),
235                // An integer beyond i64 is a float, as in JS; the float parse below
236                // cannot fail for digits alone, and it reports the error if it does.
237                Err(_) => Literal::Float(parse_float(&raw)?),
238            }
239        };
240
241        Ok(Token::new(TokenType::Number, raw, literal))
242    }
243
244    fn read_identifier(&mut self) -> Token {
245        let start = self.position;
246
247        while is_alpha_numeric(self.peek()) {
248            self.advance();
249        }
250
251        let value = self.slice(start, self.position);
252
253        match value.as_str() {
254            "true" => Token::new(TokenType::Boolean, value, Literal::Bool(true)),
255            "false" => Token::new(TokenType::Boolean, value, Literal::Bool(false)),
256            "null" => Token::new(TokenType::Null, value, Literal::None),
257            "in" => Token::new(TokenType::In, value, Literal::None),
258            "not" => {
259                // Lookahead for "not in" (whitespace then 'in')
260                let saved = self.position;
261                self.match_whitespace();
262                if self.match_string("in") {
263                    return Token::new(TokenType::NotIn, "not in", Literal::None);
264                }
265                self.position = saved;
266                Token::new(TokenType::Identifier, value.clone(), Literal::Str(value))
267            }
268            _ => Token::new(TokenType::Identifier, value.clone(), Literal::Str(value)),
269        }
270    }
271
272    fn match_whitespace(&mut self) -> bool {
273        let mut matched = false;
274        while !self.is_at_end() && is_whitespace(self.peek()) {
275            self.advance();
276            matched = true;
277        }
278        matched
279    }
280
281    fn match_string(&mut self, s: &str) -> bool {
282        let bytes = s.as_bytes();
283        let end = self.position + bytes.len();
284        if end <= self.input.len() && &self.input[self.position..end] == bytes {
285            self.position = end;
286            return true;
287        }
288        false
289    }
290
291    fn slice(&self, start: usize, end: usize) -> String {
292        // Tokens are ASCII-bounded; from_utf8_lossy is exact for our byte spans.
293        String::from_utf8_lossy(&self.input[start..end]).into_owned()
294    }
295
296    fn peek(&self) -> u8 {
297        if self.position < self.input.len() {
298            self.input[self.position]
299        } else {
300            0
301        }
302    }
303
304    fn peek_next(&self) -> u8 {
305        if self.position + 1 < self.input.len() {
306            self.input[self.position + 1]
307        } else {
308            0
309        }
310    }
311
312    fn advance(&mut self) -> u8 {
313        let ch = self.peek();
314        self.position += 1;
315        ch
316    }
317
318    fn is_at_end(&self) -> bool {
319        self.position >= self.input.len()
320    }
321}
322
323/// The float value of number text; text such as a bare exponent `1e` is an error.
324fn parse_float(raw: &str) -> Result<f64, ParseError> {
325    raw.parse::<f64>()
326        .map_err(|error| ParseError::new(format!("invalid number `{raw}`: {error}")))
327}
328
329fn is_digit(c: u8) -> bool {
330    c.is_ascii_digit()
331}
332
333fn is_alpha(c: u8) -> bool {
334    c.is_ascii_alphabetic() || c == b'_'
335}
336
337fn is_alpha_numeric(c: u8) -> bool {
338    is_alpha(c) || is_digit(c)
339}
340
341fn is_whitespace(c: u8) -> bool {
342    matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c)
343}
polyspec_crudui_validator/expr/lexer.rs

polyspec_crudui_validator/expr/
lexer.rs
