glsl_lang_pp/lexer/pre/
token.rs

1#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
3#[repr(u16)]
4pub enum Token {
5    /// Identifier or keyword
6    IDENT_KW = 1,
7    /// defined preprocessor keyword
8    DEFINED = 2,
9    /// Digit sequence
10    DIGITS = 3,
11    /// .
12    PERIOD = 4,
13    /// +
14    PLUS = 5,
15    /// -
16    DASH = 6,
17    /// /
18    SLASH = 7,
19    /// *
20    ASTERISK = 8,
21    /// %
22    PERCENT = 9,
23    /// <
24    LANGLE = 10,
25    /// >
26    RANGLE = 11,
27    /// [
28    LBRACKET = 12,
29    /// ]
30    RBRACKET = 13,
31    /// (
32    LPAREN = 14,
33    /// )
34    RPAREN = 15,
35    /// {
36    LBRACE = 16,
37    /// }
38    RBRACE = 17,
39    /// ^
40    CARET = 18,
41    /// |
42    BAR = 19,
43    /// &
44    AMPERSAND = 20,
45    /// ~
46    TILDE = 21,
47    /// =
48    EQUAL = 22,
49    /// !
50    BANG = 23,
51    /// :
52    COLON = 24,
53    /// ;
54    SEMICOLON = 25,
55    /// ,
56    COMMA = 26,
57    /// ?
58    QUESTION = 27,
59    /// #
60    HASH = 28,
61    /// "string"
62    QUOTE_STRING = 29,
63    /// <string>
64    ANGLE_STRING = 30,
65    /// \
66    BACKSLASH = 31,
67    /// Whitespace
68    WS = 32,
69    /// Newline
70    NEWLINE = 33,
71    /// Comment (single-line or multi-line)
72    COMMENT = 34,
73    /// Line continuation (required for tracking exact offsets)
74    LINECONT = 35,
75    /// Invalid token
76    ERROR = 36,
77}
78
79impl Token {
80    pub fn is_trivia(&self) -> bool {
81        matches!(self, Self::LINECONT | Self::WS | Self::COMMENT)
82    }
83
84    pub fn from_punct(raw: &str) -> Self {
85        use Token::*;
86
87        match raw {
88            "." => PERIOD,
89            "+" => PLUS,
90            "-" => DASH,
91            "/" => SLASH,
92            "*" => ASTERISK,
93            "%" => PERCENT,
94            "<" => LANGLE,
95            ">" => RANGLE,
96            "[" => LBRACKET,
97            "]" => RBRACKET,
98            "(" => LPAREN,
99            ")" => RPAREN,
100            "{" => LBRACE,
101            "}" => RBRACE,
102            "^" => CARET,
103            "|" => BAR,
104            "&" => AMPERSAND,
105            "~" => TILDE,
106            "=" => EQUAL,
107            "!" => BANG,
108            ":" => COLON,
109            ";" => SEMICOLON,
110            "," => COMMA,
111            "?" => QUESTION,
112            "#" => HASH,
113            _ => ERROR,
114        }
115    }
116}
glsl_lang_pp/lexer/pre/token.rs

glsl_lang_pp/lexer/pre/
token.rs
