polyspec_crudui_validator/expr/
token.rs

1//! CRUDUI expression tokens (expressions.md §1, JS `types.ts` TokenType parity).
2//!
3//! `TokenType` is the terminal tag; `Token` is `{type,value,literal}`. WHITESPACE
4//! never reaches the token array (the lexer drops it); EOF is always the final
5//! token. The serialized shape (`to_value`) byte-matches the shared fixture's
6//! `tokens` entries — no `position` field, since the 4-language token contract
7//! excludes positions (the Rust AST holds none either).
8//!
9//! Intentionally absent (GRAMMAR §1·§10): arithmetic `+ - * / %`, function calls,
10//! method calls, regex, assignment `=`, bitwise, root path `/`. Do not add them.
11
12use serde_json::{json, Value};
13
14/// Terminal token kind. The string spelling matches the shared fixture's
15/// `token.type` ("DOT"|"IDENTIFIER"|…).
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub enum TokenType {
18    /// Quoted string literal `'…'` / `"…"`.
19    String,
20    /// Integer or decimal number literal.
21    Number,
22    /// `true` / `false`.
23    Boolean,
24    /// `null`.
25    Null,
26    /// Path segment / unquoted identifier `[A-Za-z_][A-Za-z0-9_]*`.
27    Identifier,
28    /// Single leading `.`.
29    Dot,
30    /// Two-or-more dots `..` / `…` (parent ascent).
31    DotDot,
32    /// Wildcard `*`.
33    Asterisk,
34    /// `==`.
35    Eq,
36    /// `!=`.
37    Ne,
38    /// `>`.
39    Gt,
40    /// `>=`.
41    Ge,
42    /// `<`.
43    Lt,
44    /// `<=`.
45    Le,
46    /// `&&`.
47    And,
48    /// `||`.
49    Or,
50    /// `!`.
51    Not,
52    /// `in`.
53    In,
54    /// `not in`.
55    NotIn,
56    /// `(`.
57    LParen,
58    /// `)`.
59    RParen,
60    /// `[`.
61    LBracket,
62    /// `]`.
63    RBracket,
64    /// `,`.
65    Comma,
66    /// `?`.
67    Question,
68    /// `:`.
69    Colon,
70    /// End of input.
71    Eof,
72    /// Whitespace run (dropped before the token array is built).
73    Whitespace,
74    /// One unrecognized character.
75    Invalid,
76}
77
78impl TokenType {
79    /// The fixture string tag for this token type.
80    pub fn as_str(self) -> &'static str {
81        match self {
82            TokenType::String => "STRING",
83            TokenType::Number => "NUMBER",
84            TokenType::Boolean => "BOOLEAN",
85            TokenType::Null => "NULL",
86            TokenType::Identifier => "IDENTIFIER",
87            TokenType::Dot => "DOT",
88            TokenType::DotDot => "DOT_DOT",
89            TokenType::Asterisk => "ASTERISK",
90            TokenType::Eq => "EQ",
91            TokenType::Ne => "NE",
92            TokenType::Gt => "GT",
93            TokenType::Ge => "GE",
94            TokenType::Lt => "LT",
95            TokenType::Le => "LE",
96            TokenType::And => "AND",
97            TokenType::Or => "OR",
98            TokenType::Not => "NOT",
99            TokenType::In => "IN",
100            TokenType::NotIn => "NOT_IN",
101            TokenType::LParen => "LPAREN",
102            TokenType::RParen => "RPAREN",
103            TokenType::LBracket => "LBRACKET",
104            TokenType::RBracket => "RBRACKET",
105            TokenType::Comma => "COMMA",
106            TokenType::Question => "QUESTION",
107            TokenType::Colon => "COLON",
108            TokenType::Eof => "EOF",
109            TokenType::Whitespace => "WHITESPACE",
110            TokenType::Invalid => "INVALID",
111        }
112    }
113}
114
115/// A literal payload carried by a value token. `value` for STRING/NUMBER/
116/// BOOLEAN/NULL, the dot count for DOT_DOT, the identifier text for IDENTIFIER;
117/// `None` for everything else.
118#[derive(Debug, Clone, PartialEq)]
119pub enum Literal {
120    /// Decoded string content (STRING) or identifier text (IDENTIFIER).
121    Str(String),
122    /// Integer number literal (the fixture keeps integers as integers).
123    Int(i64),
124    /// Decimal number literal.
125    Float(f64),
126    /// Boolean keyword literal.
127    Bool(bool),
128    /// Dot count for a DOT_DOT token.
129    DotCount(i64),
130    /// No literal payload.
131    None,
132}
133
134impl Literal {
135    fn to_value(&self) -> Value {
136        match self {
137            Literal::Str(s) => Value::String(s.clone()),
138            Literal::Int(n) => json!(n),
139            Literal::Float(f) => json!(f),
140            Literal::Bool(b) => Value::Bool(*b),
141            Literal::DotCount(n) => json!(n),
142            Literal::None => Value::Null,
143        }
144    }
145}
146
147/// A single lexical token. `value` is the raw source text; `literal` is the
148/// parsed payload (or `None`). No position field — the 4-language token contract
149/// excludes positions, matching the shared fixture's token shape.
150#[derive(Debug, Clone, PartialEq)]
151pub struct Token {
152    /// The token kind.
153    pub token_type: TokenType,
154    /// Raw source text for this token.
155    pub value: String,
156    /// Parsed literal payload, if any.
157    pub literal: Literal,
158}
159
160impl Token {
161    /// Build a token from its type, raw text, and literal payload.
162    pub fn new(token_type: TokenType, value: impl Into<String>, literal: Literal) -> Self {
163        Token {
164            token_type,
165            value: value.into(),
166            literal,
167        }
168    }
169
170    /// Serialize to the shared-fixture token shape `{type,value,literal}`.
171    pub fn to_value(&self) -> Value {
172        json!({
173            "type": self.token_type.as_str(),
174            "value": self.value,
175            "literal": self.literal.to_value(),
176        })
177    }
178}
polyspec_crudui_validator/expr/token.rs

polyspec_crudui_validator/expr/
token.rs
