glsl_lang_pp/processor/
fs.rs

1use std::{
2    collections::{hash_map::Entry, HashMap},
3    path::{Path, PathBuf},
4};
5
6use bimap::BiHashMap;
7
8use lang_util::{
9    located::{FileIdResolver, Located, LocatedBuilder},
10    FileId,
11};
12
13use crate::{
14    last::LocatedIterator,
15    parser::{Ast, Parser},
16    types::path::{ParsedPath, PathType},
17};
18
19use super::{
20    event::{Event, ProcessingErrorKind},
21    expand::{ExpandEvent, ExpandOne},
22    ProcessorState,
23};
24
25pub trait FileSystem {
26    type Error: std::error::Error + 'static;
27
28    fn canonicalize(&self, path: &Path) -> Result<PathBuf, Self::Error>;
29    fn exists(&self, path: &Path) -> bool;
30    fn read(&self, path: &Path) -> Result<std::borrow::Cow<'_, str>, Self::Error>;
31}
32
33#[derive(Default, Debug, Clone, Copy)]
34pub struct Std;
35
36impl FileSystem for Std {
37    type Error = std::io::Error;
38
39    fn canonicalize(&self, path: &Path) -> Result<PathBuf, Self::Error> {
40        std::fs::canonicalize(path)
41    }
42
43    fn exists(&self, path: &Path) -> bool {
44        path.exists()
45    }
46
47    fn read(&self, path: &Path) -> Result<std::borrow::Cow<'_, str>, Self::Error> {
48        std::fs::read_to_string(path).map(Into::into)
49    }
50}
51
52pub type StdProcessor = Processor<Std>;
53
54pub struct ExpandStack<'p, F: FileSystem> {
55    processor: &'p mut Processor<F>,
56    stack: Vec<ExpandOne>,
57    state: Option<ProcessorState>,
58}
59
60impl<'p, F: FileSystem> ExpandStack<'p, F> {
61    pub fn tokenize(
62        self,
63        current_version: u16,
64        target_vulkan: bool,
65        registry: &crate::exts::Registry,
66    ) -> crate::last::Tokenizer<'_, Self> {
67        crate::last::Tokenizer::new(self, current_version, target_vulkan, registry)
68    }
69
70    pub fn into_state(self) -> Option<ProcessorState> {
71        self.state
72    }
73}
74
75impl<'p, F: FileSystem> Iterator for ExpandStack<'p, F> {
76    type Item = Result<Event, Located<F::Error>>;
77
78    fn next(&mut self) -> Option<Self::Item> {
79        loop {
80            if let Some(mut expand) = self.stack.pop() {
81                let result = expand.next();
82
83                if let Some(event) = result {
84                    match event {
85                        ExpandEvent::Event(event) => {
86                            // Put it back on the stack
87                            self.stack.push(expand);
88
89                            return Some(match event {
90                                Event::EnterFile { file_id, .. } => {
91                                    if let Some((canonical_path, input_path)) =
92                                        self.processor.get_paths(file_id)
93                                    {
94                                        Ok(Event::EnterFile {
95                                            file_id,
96                                            path: input_path.to_owned(),
97                                            canonical_path: canonical_path.to_owned(),
98                                        })
99                                    } else {
100                                        // Source block, no file path available
101                                        Ok(Event::EnterFile {
102                                            file_id,
103                                            path: Default::default(),
104                                            canonical_path: Default::default(),
105                                        })
106                                    }
107                                }
108                                other => Ok(other),
109                            });
110                        }
111                        ExpandEvent::EnterFile(node, path) => {
112                            let state = expand.state().unwrap().clone();
113
114                            // Put it back on the stack
115                            self.stack.push(expand);
116
117                            // Get the location
118                            let location = self.stack.last().unwrap().location();
119
120                            // We are supposed to enter a new file
121                            // First, parse it using the preprocessor
122                            if let Some(resolved_path) = self
123                                .processor
124                                .resolve_relative_to_id(location.current_file(), &path)
125                            {
126                                // TODO: Allow passing an encoding from somewhere
127                                match self.processor.parse(&resolved_path) {
128                                    Ok(parsed) => {
129                                        self.stack.push(parsed.expand_one(state));
130                                    }
131                                    Err(error) => {
132                                        // Just return the error, we'll keep iterating on the lower
133                                        // file by looping
134                                        return Some(Err(LocatedBuilder::new()
135                                            .pos(node.text_range())
136                                            .path(resolved_path)
137                                            .resolve_file(location)
138                                            .finish(error)));
139                                    }
140                                }
141                            } else {
142                                // Resolving the path failed, throw an error located at the
143                                // right place
144                                return Some(Ok(Event::error(
145                                    ProcessingErrorKind::IncludeNotFound { path },
146                                    node.text_range(),
147                                    location,
148                                    false,
149                                )));
150                            }
151                        }
152                        ExpandEvent::Completed(state) => {
153                            if let Some(last) = self.stack.last_mut() {
154                                // Propagate the updated state upwards in the stack
155                                last.set_state(state);
156                            } else {
157                                // No more, store the final state
158                                self.state = Some(state);
159                            }
160                        }
161                    }
162                }
163            } else {
164                return None;
165            }
166        }
167    }
168}
169
170impl<'p, F: FileSystem> FileIdResolver for ExpandStack<'p, F> {
171    fn resolve(&self, file_id: FileId) -> Option<&Path> {
172        <Processor<F> as FileIdResolver>::resolve(self.processor, file_id)
173    }
174}
175
176#[derive(Debug)]
177pub struct ParsedFile<'p, F: FileSystem> {
178    processor: &'p mut Processor<F>,
179    file_id: FileId,
180}
181
182impl<'p, F: FileSystem> ParsedFile<'p, F> {
183    pub fn file_id(&self) -> FileId {
184        self.file_id
185    }
186
187    pub fn process(self, initial_state: ProcessorState) -> ExpandStack<'p, F> {
188        let ast = self.ast();
189
190        ExpandStack {
191            processor: self.processor,
192            stack: vec![ExpandOne::new((self.file_id, ast), initial_state)],
193            state: None,
194        }
195    }
196
197    pub fn ast(&self) -> Ast {
198        self.processor
199            .file_cache
200            .get(&self.file_id)
201            .unwrap()
202            .clone()
203    }
204
205    fn expand_one(self, initial_state: ProcessorState) -> ExpandOne {
206        ExpandOne::new(self, initial_state)
207    }
208}
209
210impl<'p, F: FileSystem> From<ParsedFile<'p, F>> for (FileId, Ast) {
211    fn from(parsed_file: ParsedFile<'p, F>) -> Self {
212        let ast = parsed_file.ast();
213        (parsed_file.file_id, ast)
214    }
215}
216
217impl<'p, F: FileSystem> IntoIterator for ParsedFile<'p, F> {
218    type Item = <ExpandStack<'p, F> as Iterator>::Item;
219    type IntoIter = ExpandStack<'p, F>;
220
221    fn into_iter(self) -> Self::IntoIter {
222        self.process(ProcessorState::default())
223    }
224}
225
226#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::From)]
227enum PathOrSource {
228    Source(usize, PathBuf),
229    Path(PathBuf),
230}
231
232impl PathOrSource {
233    fn as_path(&self) -> Option<&PathBuf> {
234        match self {
235            PathOrSource::Source(_, _) => None,
236            PathOrSource::Path(path) => Some(path),
237        }
238    }
239}
240
241/// Preprocessor based on a filesystem
242#[derive(Debug)]
243pub struct Processor<F: FileSystem> {
244    /// Cache of parsed files (preprocessor token sequences)
245    file_cache: HashMap<FileId, Ast>,
246    /// Mapping from canonical paths to FileIds
247    file_ids: BiHashMap<PathOrSource, FileId>,
248    /// Mapping from #include/input paths to canonical paths
249    canonical_paths: BiHashMap<PathBuf, PathBuf>,
250    /// List of include paths in resolution order
251    system_paths: Vec<PathBuf>,
252    /// Filesystem abstraction
253    fs: F,
254}
255
256impl<F: FileSystem> Processor<F> {
257    pub fn new_with_fs(fs: F) -> Self {
258        Self {
259            file_cache: HashMap::with_capacity(1),
260            file_ids: BiHashMap::with_capacity(1),
261            canonical_paths: BiHashMap::with_capacity(1),
262            system_paths: Vec::new(),
263            fs,
264        }
265    }
266
267    fn get_paths(&self, file_id: FileId) -> Option<(&PathBuf, &PathBuf)> {
268        // Find the canonical path for the current file identifier
269        let canonical_path = self.file_ids.get_by_right(&file_id)?;
270
271        let canonical_path = match canonical_path {
272            PathOrSource::Source(_, _) => {
273                return None;
274            }
275            PathOrSource::Path(path) => path,
276        };
277
278        // Transform that back into a non-canonical path
279        let input_path = self.canonical_paths.get_by_right(canonical_path)?;
280
281        Some((canonical_path, input_path))
282    }
283
284    pub fn system_paths(&self) -> &Vec<PathBuf> {
285        &self.system_paths
286    }
287
288    pub fn system_paths_mut(&mut self) -> &mut Vec<PathBuf> {
289        &mut self.system_paths
290    }
291
292    fn resolve_relative_to_id(&self, relative_to: FileId, path: &ParsedPath) -> Option<PathBuf> {
293        let parent = self
294            .file_ids
295            .get_by_right(&relative_to)
296            .and_then(|key| match key {
297                PathOrSource::Source(_, dir) => Some(dir.as_path()),
298                PathOrSource::Path(path) => self
299                    .canonical_paths
300                    .get_by_right(path)
301                    .and_then(|path| path.parent()),
302            })?;
303
304        self.resolve_relative_to_path(parent, path)
305    }
306
307    pub fn resolve_relative_to_path(
308        &self,
309        parent: impl AsRef<Path>,
310        path: &ParsedPath,
311    ) -> Option<PathBuf> {
312        // Convert the path (string) to a pathbuf
313        let path_as_pathbuf = PathBuf::from(&path.path);
314
315        if path_as_pathbuf.is_absolute() {
316            // If it's absolute, return it as-is
317            return Some(path_as_pathbuf);
318        }
319
320        // Else, try to resolve it
321        match path.ty {
322            PathType::Angle => self.system_paths.iter().find_map(|system_path| {
323                let full_path = system_path.join(&path_as_pathbuf);
324                self.fs.exists(&full_path).then_some(full_path)
325            }),
326            PathType::Quote => Some(parent.as_ref().join(path_as_pathbuf)),
327        }
328    }
329
330    pub fn parse(&mut self, path: &Path) -> Result<ParsedFile<F>, F::Error> {
331        // Find the canonical path. Not using the entry API because cloning a path is expensive.
332        let canonical_path = if let Some(canonical_path) = self.canonical_paths.get_by_left(path) {
333            canonical_path
334        } else {
335            let canonical_path = self.fs.canonicalize(path)?;
336            self.canonical_paths.insert(path.to_owned(), canonical_path);
337            self.canonical_paths.get_by_left(path).unwrap()
338        };
339
340        // Allocate a file id. Not using the entry API because cloning a path is expensive.
341        let key: PathOrSource = canonical_path.to_owned().into();
342        let file_id = if let Some(file_id) = self.file_ids.get_by_left(&key) {
343            *file_id
344        } else {
345            let file_id = FileId::new(self.file_ids.len() as _);
346            self.file_ids.insert(key, file_id);
347            file_id
348        };
349
350        match self.file_cache.entry(file_id) {
351            Entry::Occupied(_) => Ok(ParsedFile {
352                processor: self,
353                file_id,
354            }),
355            Entry::Vacant(entry) => {
356                // Read the file
357                let input = self.fs.read(canonical_path)?;
358                // Parse it
359                let ast = Parser::new(&input).parse();
360                // Check that the root node covers the entire range
361                debug_assert_eq!(u32::from(ast.green_node().text_len()), input.len() as u32);
362                // Insert it
363                entry.insert(ast);
364
365                Ok(ParsedFile {
366                    processor: self,
367                    file_id,
368                })
369            }
370        }
371    }
372
373    /// Parse a given source block as if it belonged in a specific directory
374    ///
375    /// # Parameters
376    ///
377    /// * `source`: GLSL source block to parse
378    /// * `path`: path to the directory that (virtually) contains this GLSL source block
379    pub fn parse_source(&mut self, source: &str, path: &Path) -> ParsedFile<F> {
380        // Create key for this source block
381        let key = PathOrSource::Source(self.file_ids.len(), path.to_owned());
382
383        // Register file id
384        let file_id = FileId::new(self.file_ids.len() as _);
385        self.file_ids.insert(key, file_id);
386
387        // Parse the source
388        let ast = Parser::new(source).parse();
389        // Check that the root node covers the entire range
390        debug_assert_eq!(u32::from(ast.green_node().text_len()), source.len() as u32);
391        // Insert into the cache
392        self.file_cache.insert(file_id, ast);
393
394        ParsedFile {
395            processor: self,
396            file_id,
397        }
398    }
399}
400
401impl<F: FileSystem + Default> Processor<F> {
402    pub fn new() -> Self {
403        Self::default()
404    }
405}
406
407impl<F: FileSystem + Default> Default for Processor<F> {
408    fn default() -> Self {
409        Self::new_with_fs(F::default())
410    }
411}
412
413impl<F: FileSystem> FileIdResolver for Processor<F> {
414    fn resolve(&self, file_id: FileId) -> Option<&Path> {
415        self.file_ids
416            .get_by_right(&file_id)
417            .and_then(PathOrSource::as_path)
418            .and_then(|canonical_path| self.canonical_paths.get_by_right(canonical_path))
419            .map(|pathbuf| pathbuf.as_path())
420    }
421}
422
423impl<'p, F: FileSystem> LocatedIterator for ExpandStack<'p, F> {
424    fn location(&self) -> &crate::processor::expand::ExpandLocation {
425        self.stack.last().unwrap().location()
426    }
427}
glsl_lang_pp/processor/fs.rs

glsl_lang_pp/processor/
fs.rs
