glsl_lang_lexer/full/
fs.rs

1//! Filesystem based glsl-lang-pp preprocessing lexer
2
3use std::path::{Path, PathBuf};
4
5use lang_util::position::LexerPosition;
6
7use glsl_lang_pp::{
8    exts::{Registry, DEFAULT_REGISTRY},
9    last::{self, Event},
10    processor::{
11        fs::{ExpandStack, ParsedFile, Processor},
12        ProcessorState,
13    },
14};
15
16use crate::{HasLexerError, LangLexer, LangLexerIterator, ParseContext, ParseOptions, Token};
17
18use super::{
19    core::{self, HandleTokenResult, LexerCore},
20    Directives, LexicalError,
21};
22
23pub use glsl_lang_pp::processor::fs::FileSystem;
24
25/// glsl-lang-pp filesystem lexer
26pub struct Lexer<'r, 'p, F: FileSystem> {
27    inner: last::Tokenizer<'r, ExpandStack<'p, F>>,
28    current_file: PathBuf,
29    handle_token: HandleTokenResult<F::Error>,
30    opts: ParseOptions,
31}
32
33impl<'r, 'p, F: FileSystem> Lexer<'r, 'p, F> {
34    fn new(inner: ExpandStack<'p, F>, registry: &'r Registry, opts: &ParseOptions) -> Self {
35        Self {
36            inner: inner.tokenize(opts.default_version, opts.target_vulkan, registry),
37            current_file: Default::default(),
38            handle_token: Default::default(),
39            opts: *opts,
40        }
41    }
42
43    fn with_context(self, ctx: ParseContext) -> LexerIterator<'r, 'p, F> {
44        LexerIterator {
45            inner: self.inner,
46            core: LexerCore::new(&self.opts, ctx),
47            current_file: self.current_file,
48            handle_token: self.handle_token,
49        }
50    }
51}
52
53/// glsl-lang-pp filesystem lexer iterator
54pub struct LexerIterator<'r, 'p, F: FileSystem> {
55    inner: last::Tokenizer<'r, ExpandStack<'p, F>>,
56    core: LexerCore,
57    current_file: PathBuf,
58    handle_token: HandleTokenResult<F::Error>,
59}
60
61impl<F: FileSystem> LexerIterator<'_, '_, F> {
62    pub fn into_directives(self) -> Directives {
63        self.core.into_directives()
64    }
65}
66
67impl<'r, 'p, F: FileSystem> Iterator for LexerIterator<'r, 'p, F> {
68    type Item = core::Item<F::Error>;
69
70    fn next(&mut self) -> Option<Self::Item> {
71        loop {
72            // Pop pending events
73            if let Some(item) = self.handle_token.pop_item() {
74                return Some(item);
75            }
76
77            if let Some(result) = self.handle_token.pop_event().or_else(|| self.inner.next()) {
78                match result {
79                    Ok(event) => match event {
80                        Event::Error { error, masked } => {
81                            if !masked {
82                                return Some(Err(error.into()));
83                            }
84                        }
85
86                        Event::Token {
87                            source_token,
88                            token_kind,
89                            state,
90                        } => {
91                            self.core.handle_token(
92                                source_token,
93                                token_kind,
94                                state,
95                                &mut self.inner,
96                                &mut self.handle_token,
97                            );
98                        }
99
100                        Event::Directive { directive, masked } => {
101                            if let Err(errors) = self.core.handle_directive(directive, masked) {
102                                self.handle_token.push_errors(errors);
103                            }
104                        }
105
106                        Event::EnterFile {
107                            file_id,
108                            path,
109                            canonical_path: _,
110                        } => {
111                            self.current_file = path;
112                            self.core.handle_file_id(file_id);
113                        }
114                    },
115
116                    Err(err) => {
117                        return Some(Err(LexicalError::Io(err)));
118                    }
119                }
120            } else {
121                return None;
122            }
123        }
124    }
125}
126
127impl<F: FileSystem> HasLexerError for Lexer<'_, '_, F> {
128    type Error = LexicalError<F::Error>;
129}
130
131impl<'r, 'p, F: FileSystem> LangLexer<'p> for Lexer<'r, 'p, F>
132where
133    File<'r, 'p, F>: 'p,
134{
135    type Input = File<'r, 'p, F>;
136    type Iter = LexerIterator<'r, 'p, F>;
137
138    fn new(source: Self::Input, opts: &ParseOptions) -> Self {
139        Lexer::new(
140            source.inner.process(source.state.unwrap_or_default()),
141            source.registry.unwrap_or(&DEFAULT_REGISTRY),
142            opts,
143        )
144    }
145
146    fn run(self, ctx: ParseContext) -> Self::Iter {
147        self.with_context(ctx)
148    }
149}
150
151impl<F: FileSystem> HasLexerError for LexerIterator<'_, '_, F> {
152    type Error = LexicalError<F::Error>;
153}
154
155impl<'r, 'p, F: FileSystem> LangLexerIterator for LexerIterator<'r, 'p, F> {
156    fn resolve_err(
157        &self,
158        err: lalrpop_util::ParseError<LexerPosition, Token, Self::Error>,
159    ) -> lang_util::error::ParseError<Self::Error> {
160        let location = self.inner.location();
161        let (file_id, lexer) = lang_util::error::error_location(&err);
162
163        lang_util::error::ParseError::<Self::Error>::builder()
164            .pos(lexer)
165            .current_file(file_id)
166            .resolve(location)
167            .resolve_path(&self.inner)
168            .finish(err.into())
169    }
170}
171
172/// glsl-lang-pp preprocessor extensions
173pub trait PreprocessorExt<F: FileSystem> {
174    /// Open the given file for lexing
175    ///
176    /// # Parameters
177    ///
178    /// * `path`: path to the file to open
179    fn open(&mut self, path: impl AsRef<Path>) -> Result<File<'_, '_, F>, F::Error>;
180
181    /// Open the given source block for lexing
182    ///
183    /// # Parameters
184    ///
185    /// * `source`: source string to parse
186    /// * `path`: path to the directory that contains this source
187    fn open_source(&mut self, source: &str, path: impl AsRef<Path>) -> File<'_, '_, F>;
188}
189
190impl<F: FileSystem> PreprocessorExt<F> for Processor<F> {
191    fn open(&mut self, path: impl AsRef<Path>) -> Result<File<'_, '_, F>, F::Error> {
192        self.parse(path.as_ref()).map(|parsed_file| File {
193            inner: parsed_file,
194            state: None,
195            registry: None,
196        })
197    }
198
199    fn open_source(&mut self, source: &str, path: impl AsRef<Path>) -> File<'_, '_, F> {
200        File {
201            inner: self.parse_source(source, path.as_ref()),
202            state: None,
203            registry: None,
204        }
205    }
206}
207
208/// A preprocessor parsed file ready for lexing
209pub struct File<'r, 'p, F: FileSystem> {
210    inner: ParsedFile<'p, F>,
211    state: Option<ProcessorState>,
212    registry: Option<&'r Registry>,
213}
214
215impl<'r, 'p, F: FileSystem> File<'r, 'p, F> {
216    /// Set the default processor state for processing this file
217    pub fn with_state(self, state: impl Into<ProcessorState>) -> Self {
218        Self {
219            state: Some(state.into()),
220            ..self
221        }
222    }
223
224    /// Set the extension registry to use for this file
225    pub fn with_registry(self, registry: impl Into<&'r Registry>) -> Self {
226        Self {
227            registry: Some(registry.into()),
228            ..self
229        }
230    }
231}
glsl_lang_lexer/full/fs.rs

glsl_lang_lexer/full/
fs.rs
