glsl_lang_lexer/full/
core.rs

1use std::collections::VecDeque;
2
3use glsl_lang_pp::{
4    last::{self, LocatedIterator, MaybeToken, TokenState, Tokenizer},
5    processor::event::{self, Error, EventDirective, OutputToken},
6    types,
7};
8
9use glsl_lang_types::ast;
10
11use lang_util::{located::Located, position::NodeSpan, FileId, NodeContent, TextRange};
12
13use crate::{ParseContext, ParseOptions};
14
15use super::{Directives, LexerPosition, LexicalError, Token};
16
17pub type Item<E> = Result<(LexerPosition, Token, LexerPosition), LexicalError<E>>;
18
19pub struct LexerCore {
20    pub ctx: ParseContext,
21    file_id: FileId,
22    opts: ParseOptions,
23    directives: Vec<EventDirective>,
24}
25
26pub enum HandleTokenResult<E: std::error::Error + 'static> {
27    None,
28    Item(Item<E>),
29    Pending(VecDeque<Item<E>>, VecDeque<Result<last::Event, Located<E>>>),
30}
31
32impl<E: std::error::Error + 'static> HandleTokenResult<E> {
33    pub fn push_item(&mut self, item: Item<E>) {
34        match std::mem::take(self) {
35            HandleTokenResult::None => {
36                *self = HandleTokenResult::Item(item);
37            }
38            HandleTokenResult::Item(old_item) => {
39                let mut pending_items = VecDeque::with_capacity(2);
40                pending_items.push_back(old_item);
41                pending_items.push_back(item);
42                *self = HandleTokenResult::Pending(pending_items, VecDeque::new());
43            }
44            HandleTokenResult::Pending(mut items, events) => {
45                items.push_back(item);
46                *self = HandleTokenResult::Pending(items, events);
47            }
48        }
49    }
50
51    pub fn pop_item(&mut self) -> Option<Item<E>> {
52        match std::mem::take(self) {
53            HandleTokenResult::None => None,
54            HandleTokenResult::Item(item) => Some(item),
55            HandleTokenResult::Pending(mut items, events) => {
56                let item = items.pop_front();
57                if !items.is_empty() || !events.is_empty() {
58                    *self = Self::Pending(items, events);
59                }
60                item
61            }
62        }
63    }
64
65    pub fn pop_event(&mut self) -> Option<Result<last::Event, Located<E>>> {
66        match std::mem::take(self) {
67            HandleTokenResult::None => None,
68            HandleTokenResult::Item(item) => {
69                *self = Self::Item(item);
70                None
71            }
72            HandleTokenResult::Pending(items, mut events) => {
73                let event = events.pop_front();
74                if !items.is_empty() || !events.is_empty() {
75                    *self = Self::Pending(items, events);
76                }
77                event
78            }
79        }
80    }
81
82    pub fn push_errors(&mut self, errors: impl IntoIterator<Item = Error>) {
83        let items = errors.into_iter().map(|error| Err(error.into()));
84
85        match std::mem::take(self) {
86            HandleTokenResult::None => {
87                *self = HandleTokenResult::Pending(items.collect(), Default::default());
88            }
89            HandleTokenResult::Item(old_item) => {
90                let mut pending_items = VecDeque::with_capacity(items.size_hint().0 + 1);
91                pending_items.push_back(old_item);
92                pending_items.extend(items);
93                *self = HandleTokenResult::Pending(pending_items, Default::default());
94            }
95            HandleTokenResult::Pending(mut old_items, events) => {
96                old_items.extend(items);
97                *self = HandleTokenResult::Pending(old_items, events);
98            }
99        }
100    }
101}
102
103impl<E: std::error::Error + 'static> Default for HandleTokenResult<E> {
104    fn default() -> Self {
105        Self::None
106    }
107}
108
109impl LexerCore {
110    pub fn new(opts: &ParseOptions, ctx: ParseContext) -> Self {
111        let file_id = opts.source_id;
112        Self {
113            ctx,
114            file_id,
115            opts: *opts,
116            directives: Vec::with_capacity(2),
117        }
118    }
119
120    fn lang_token(
121        &self,
122        source_token: &OutputToken,
123        token_kind: types::Token,
124    ) -> Result<(LexerPosition, Token, LexerPosition), (types::Token, types::token::ErrorKind)>
125    {
126        crate::lang_token::lang_token(
127            &self.ctx,
128            source_token.text(),
129            source_token.text_range(),
130            token_kind,
131        )
132    }
133
134    pub fn handle_file_id(&mut self, file_id: FileId) {
135        self.file_id = file_id;
136    }
137
138    pub fn handle_token<'r, I, E>(
139        &self,
140        source_token: OutputToken,
141        token_kind: types::Token,
142        state: TokenState,
143        tokenizer: &mut Tokenizer<'r, I>,
144        token_state: &mut HandleTokenResult<E>,
145    ) where
146        E: std::error::Error + 'static,
147        I: Iterator<Item = Result<event::Event, Located<E>>> + LocatedIterator,
148        <Tokenizer<'r, I> as Iterator>::Item: MaybeToken,
149    {
150        if state.active() {
151            match self.lang_token(&source_token, token_kind) {
152                Ok(token) => {
153                    // Try to get the next token when we encounter trivia
154                    match token.1 {
155                        Token::Whitespace => {}
156                        Token::SingleLineComment | Token::MultiLineComment => {
157                            if self.ctx.has_comments() {
158                                let mut text = source_token.text().split_at(2).1.to_string();
159
160                                let comment = match token.1 {
161                                    Token::SingleLineComment => ast::CommentData::Single(text),
162                                    Token::MultiLineComment => {
163                                        text.pop();
164                                        text.pop();
165                                        ast::CommentData::Multi(text)
166                                    }
167                                    _ => unreachable!(),
168                                }
169                                .spanned(token.0, token.2);
170
171                                self.ctx.add_comment(comment);
172                            }
173                        }
174                        _ => {
175                            if token.1 == Token::LeftBrace {
176                                self.ctx.push_scope();
177                            } else if token.1 == Token::RightBrace {
178                                self.ctx.pop_scope();
179                            }
180
181                            token_state.push_item(Ok(token));
182                        }
183                    }
184                }
185
186                Err((token_kind, error)) => {
187                    if !(self.opts.allow_rs_ident
188                        && error == types::token::ErrorKind::InvalidToken
189                        && token_kind == types::Token::HASH)
190                    {
191                        token_state.push_item(Err(LexicalError::Token {
192                            kind: error,
193                            pos: source_token.text_range(),
194                        }));
195                    } else {
196                        // Try to detect #(...)
197                        let start = source_token.text_range().start();
198                        let mut end = start;
199
200                        let mut pending_items = VecDeque::new();
201                        let mut pending_events = VecDeque::new();
202
203                        pending_items.push_back(Err(LexicalError::Token {
204                            kind: types::token::ErrorKind::InvalidToken,
205                            pos: NodeSpan::new(
206                                start.source_id,
207                                TextRange::new(start.offset, end.offset),
208                            ),
209                        }));
210
211                        while let Some(maybe_lparen_result) = tokenizer.next() {
212                            // Skip whitespace
213                            if let Some(types::Token::WS) = maybe_lparen_result.as_token_kind() {
214                                pending_events.push_back(maybe_lparen_result);
215                                continue;
216                            }
217
218                            if let Some(types::Token::LPAREN) = maybe_lparen_result.as_token_kind()
219                            {
220                                // We have seen a left parenthesis
221                                // Now, consume everything until the
222                                // matching rparen
223                                let mut level = 1;
224                                let mut quoted = "#(".to_owned();
225
226                                while level > 0 {
227                                    match tokenizer.next() {
228                                        Some(result) => {
229                                            if let Some((source_token, token_kind, _)) =
230                                                result.as_token()
231                                            {
232                                                match token_kind {
233                                                    types::Token::LPAREN => level += 1,
234                                                    types::Token::RPAREN => level -= 1,
235                                                    _ => {}
236                                                }
237
238                                                if level > 0 {
239                                                    quoted.push_str(source_token.text());
240                                                } else {
241                                                    end = source_token.text_range().start();
242                                                }
243                                            }
244                                        }
245                                        None => {
246                                            // End of file: return an error. Since the same error
247                                            // we would return has been added for the # token, just
248                                            // exit the loop
249                                            *token_state = HandleTokenResult::Pending(
250                                                pending_items,
251                                                pending_events,
252                                            );
253                                            return;
254                                        }
255                                    }
256                                }
257
258                                quoted.push(')');
259                                token_state.push_item(Ok((
260                                    start,
261                                    Token::Identifier(quoted.into()),
262                                    end,
263                                )));
264                            } else {
265                                pending_events.push_back(maybe_lparen_result);
266                                *token_state =
267                                    HandleTokenResult::Pending(pending_items, pending_events);
268                            }
269
270                            return;
271                        }
272                    }
273                }
274            };
275        }
276    }
277
278    pub fn into_directives(self) -> Directives {
279        self.directives.into()
280    }
281
282    pub fn handle_directive(
283        &mut self,
284        directive: EventDirective,
285        masked: bool,
286    ) -> Result<(), Vec<Error>> {
287        if masked {
288            return Ok(());
289        }
290
291        let errors = directive.errors().to_vec();
292
293        self.directives.push(directive);
294
295        if errors.is_empty() {
296            Ok(())
297        } else {
298            Err(errors)
299        }
300    }
301}
glsl_lang_lexer/full/core.rs

glsl_lang_lexer/full/
core.rs
