glsl_lang_pp/lexer/glue/
token.rs

1#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
3#[repr(u16)]
4pub enum Token {
5    /// Identifier or keyword
6    IDENT_KW = 1,
7    /// defined preprocessor keyword
8    DEFINED = 2,
9    /// Digit sequence
10    DIGITS = 3,
11    // Single-char tokens
12    /// .
13    PERIOD = 4,
14    /// +
15    PLUS = 5,
16    /// -
17    DASH = 6,
18    /// /
19    SLASH = 7,
20    /// *
21    ASTERISK = 8,
22    /// %
23    PERCENT = 9,
24    /// <
25    LANGLE = 10,
26    /// >
27    RANGLE = 11,
28    /// [
29    LBRACKET = 12,
30    /// ]
31    RBRACKET = 13,
32    /// (
33    LPAREN = 14,
34    /// )
35    RPAREN = 15,
36    /// {
37    LBRACE = 16,
38    /// }
39    RBRACE = 17,
40    /// ^
41    CARET = 18,
42    /// |
43    BAR = 19,
44    /// &
45    AMPERSAND = 20,
46    /// ~
47    TILDE = 21,
48    /// =
49    EQUAL = 22,
50    /// !
51    BANG = 23,
52    /// :
53    COLON = 24,
54    /// ;
55    SEMICOLON = 25,
56    /// ,
57    COMMA = 26,
58    /// ?
59    QUESTION = 27,
60    /// #
61    HASH = 28,
62    // Other
63    /// "string"
64    QUOTE_STRING = 29,
65    /// <string>
66    ANGLE_STRING = 30,
67    /// \
68    BACKSLASH = 31,
69    /// Whitespace
70    WS = 32,
71    /// Newline
72    NEWLINE = 33,
73    /// Comment (single-line or multi-line)
74    COMMENT = 34,
75    /// Line continuation (required for tracking exact offsets)
76    LINECONT = 35,
77    /// Invalid token
78    ERROR = 36,
79    // Multi-char tokens
80    /// <<
81    LEFT_OP = 37,
82    /// >>
83    RIGHT_OP = 38,
84    /// ++
85    INC_OP = 39,
86    /// --
87    DEC_OP = 40,
88    /// <=
89    LE_OP = 41,
90    /// >=
91    GE_OP = 42,
92    /// ==
93    EQ_OP = 43,
94    /// !=
95    NE_OP = 44,
96    /// &&
97    AND_OP = 45,
98    /// ||
99    OR_OP = 46,
100    /// ^^
101    XOR_OP = 47,
102    /// *=
103    MUL_ASSIGN = 48,
104    /// /=
105    DIV_ASSIGN = 49,
106    /// +=
107    ADD_ASSIGN = 50,
108    /// %=
109    MOD_ASSIGN = 51,
110    /// <<=
111    LEFT_ASSIGN = 52,
112    /// >>=
113    RIGHT_ASSIGN = 53,
114    /// &=
115    AND_ASSIGN = 54,
116    /// ^=
117    XOR_ASSIGN = 55,
118    /// |=
119    OR_ASSIGN = 56,
120    /// -=
121    SUB_ASSIGN = 57,
122    /// ##
123    PP_CONCAT = 58,
124}
125
126impl Token {
127    pub fn is_trivia(&self) -> bool {
128        matches!(self, Self::LINECONT | Self::WS | Self::COMMENT)
129    }
130}
131
132impl From<crate::lexer::PreToken> for Token {
133    fn from(pre: crate::lexer::PreToken) -> Self {
134        // SAFETY: Token is a superset of PreToken
135        unsafe { std::mem::transmute(std::mem::transmute::<crate::lexer::PreToken, u16>(pre)) }
136    }
137}
glsl_lang_pp/lexer/glue/token.rs

glsl_lang_pp/lexer/glue/
token.rs
