glsl_lang_pp/
last.rs

1//! Last preprocessing stage definitions
2
3use std::{collections::HashMap, iter::FusedIterator, path::PathBuf};
4
5use lang_util::{located::FileIdResolver, FileId};
6
7use crate::{
8    exts::{names::ExtNameAtom, ExtensionSpec, Registry},
9    processor::{
10        event::{self, DirectiveKind, Error, ErrorKind, EventDirective, OutputToken, TokenLike},
11        expand::ExpandLocation,
12        nodes::{Extension, ExtensionBehavior, ExtensionName},
13    },
14    types::{
15        type_names::{TypeNameAtom, TypeNameState},
16        Token, TypeName,
17    },
18};
19
20mod token;
21
22#[derive(Debug, PartialEq)]
23pub enum Event {
24    Error {
25        error: Error,
26        masked: bool,
27    },
28    EnterFile {
29        file_id: FileId,
30        path: PathBuf,
31        canonical_path: PathBuf,
32    },
33    Token {
34        source_token: OutputToken,
35        token_kind: Token,
36        state: TokenState,
37    },
38    Directive {
39        directive: EventDirective,
40        masked: bool,
41    },
42}
43
44pub trait LocatedIterator {
45    fn location(&self) -> &ExpandLocation;
46}
47
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub enum TokenState {
50    Masked,
51    Active,
52    Warn(ExtNameAtom),
53}
54
55impl TokenState {
56    fn new(state: Option<TypeNameState>, masked: bool) -> Self {
57        if masked {
58            Self::Masked
59        } else if let Some(state) = state {
60            match state {
61                TypeNameState::WarnType(name) => Self::Warn(name),
62                _ => Self::Active,
63            }
64        } else {
65            Self::Active
66        }
67    }
68
69    pub fn active(self) -> bool {
70        matches!(self, Self::Active | Self::Warn(_))
71    }
72
73    pub fn warn(self) -> bool {
74        matches!(self, Self::Warn(_))
75    }
76}
77
78struct TypeTable<'r> {
79    type_names: HashMap<TypeNameAtom, Option<(ExtNameAtom, ExtensionBehavior)>>,
80    extensions: HashMap<ExtNameAtom, ExtensionBehavior>,
81    registry: &'r Registry,
82    target_vulkan: bool,
83    current_version: u16,
84}
85
86impl<'r> TypeTable<'r> {
87    fn new(registry: &'r Registry, current_version: u16, target_vulkan: bool) -> Self {
88        Self {
89            type_names: Default::default(),
90            extensions: Default::default(),
91            registry,
92            target_vulkan,
93            current_version,
94        }
95    }
96
97    fn is_type_name(&self, name: &TypeNameAtom) -> TypeNameState {
98        if let Some(type_def) = self.type_names.get(name) {
99            match type_def {
100                Some((name, behavior)) if *behavior == ExtensionBehavior::Warn => {
101                    TypeNameState::WarnType(name.clone())
102                }
103                _ => TypeNameState::Type,
104            }
105        } else {
106            TypeNameState::Ident
107        }
108    }
109
110    fn promote_type_name(&mut self, name: TypeNameAtom) -> bool {
111        self.type_names.insert(name, None).is_some()
112    }
113
114    fn set_extension_behavior(&mut self, spec: &ExtensionSpec, behavior: ExtensionBehavior) {
115        if behavior == ExtensionBehavior::Disable {
116            // Disable the extension
117            for type_name in spec.type_names() {
118                self.type_names.remove(type_name);
119            }
120
121            self.extensions.remove(spec.name());
122        } else {
123            // Enable the extension
124            for type_name in spec.type_names() {
125                self.type_names
126                    .insert(type_name.clone(), Some((spec.name().clone(), behavior)));
127            }
128
129            self.extensions.insert(spec.name().clone(), behavior);
130        }
131    }
132
133    fn handle_extension(&mut self, extension: &Extension) -> bool {
134        match &extension.name {
135            ExtensionName::All => {
136                if extension.behavior == ExtensionBehavior::Disable
137                    || extension.behavior == ExtensionBehavior::Warn
138                {
139                    for ext in self.registry.all() {
140                        self.set_extension_behavior(ext, extension.behavior);
141                    }
142                } else {
143                    // TODO: Handle invalid behavior for all
144                }
145            }
146            ExtensionName::Specific(name) => {
147                if let Some(spec) = self.registry.get(name) {
148                    self.set_extension_behavior(spec, extension.behavior);
149                } else {
150                    return false;
151                }
152            }
153        }
154
155        true
156    }
157
158    fn tokenize_single(
159        &self,
160        token: &impl TokenLike,
161        location: &ExpandLocation,
162    ) -> (Token, Option<TypeNameState>, Option<Error>) {
163        let (token_kind, state) =
164            token::token_from_syntax_kind(token, self.current_version, self.target_vulkan, |tn| {
165                self.is_type_name(tn)
166            });
167
168        let error = if let Some(TypeNameState::WarnType(extension)) = &state {
169            Some(
170                Error::builder()
171                    .pos(token.text_range())
172                    .resolve_file(location)
173                    .finish(ErrorKind::warn_ext_use(
174                        extension.clone(),
175                        match &token_kind {
176                            Token::TYPE_NAME(TypeName::OTHER(type_name)) => Some(type_name.clone()),
177                            _ => unreachable!(),
178                        },
179                        token.text_range(),
180                        location,
181                    )),
182            )
183        } else {
184            None
185        };
186
187        (token_kind, state, error)
188    }
189}
190
191pub trait MaybeToken {
192    fn as_token(&self) -> Option<(&OutputToken, &Token, &TokenState)>;
193
194    fn as_token_kind(&self) -> Option<&Token> {
195        self.as_token().map(|(_, kind, _)| kind)
196    }
197}
198
199impl<T: MaybeToken, E> MaybeToken for Result<T, E> {
200    fn as_token(&self) -> Option<(&OutputToken, &Token, &TokenState)> {
201        self.as_ref().ok().and_then(MaybeToken::as_token)
202    }
203}
204
205impl MaybeToken for Event {
206    fn as_token(&self) -> Option<(&OutputToken, &Token, &TokenState)> {
207        match self {
208            Event::Token {
209                source_token,
210                token_kind,
211                state,
212            } => Some((source_token, token_kind, state)),
213            _ => None,
214        }
215    }
216}
217
218pub struct Tokenizer<'r, I> {
219    inner: I,
220    type_table: TypeTable<'r>,
221    pending_error: Option<Error>,
222}
223
224impl<'r, I: LocatedIterator> Tokenizer<'r, I> {
225    pub fn new(
226        inner: impl IntoIterator<IntoIter = I>,
227        current_version: u16,
228        target_vulkan: bool,
229        registry: &'r Registry,
230    ) -> Self {
231        Self {
232            inner: inner.into_iter(),
233            type_table: TypeTable::new(registry, current_version, target_vulkan),
234            pending_error: None,
235        }
236    }
237
238    pub fn tokenize_single(
239        &self,
240        token: &impl TokenLike,
241    ) -> (Token, Option<TypeNameState>, Option<Error>) {
242        self.type_table
243            .tokenize_single(token, self.inner.location())
244    }
245
246    pub fn promote_type_name(&mut self, name: TypeNameAtom) -> bool {
247        self.type_table.promote_type_name(name)
248    }
249
250    pub fn location(&self) -> &crate::processor::expand::ExpandLocation {
251        self.inner.location()
252    }
253}
254
255impl<'r, E, I: Iterator<Item = Result<event::Event, E>> + LocatedIterator> Iterator
256    for Tokenizer<'r, I>
257{
258    type Item = Result<Event, E>;
259
260    fn next(&mut self) -> Option<Self::Item> {
261        if let Some(error) = self.pending_error.take() {
262            return Some(Ok(Event::Error {
263                error,
264                masked: false,
265            }));
266        }
267
268        self.inner.next().map(|result| match result {
269            Ok(event) => Ok(match event {
270                event::Event::Error { error, masked } => Event::Error { error, masked },
271                event::Event::EnterFile {
272                    file_id,
273                    path,
274                    canonical_path,
275                } => Event::EnterFile {
276                    file_id,
277                    path,
278                    canonical_path,
279                },
280                event::Event::Token { token, masked } => {
281                    let (token_kind, state, error) = self.tokenize_single(&token);
282
283                    if !masked {
284                        self.pending_error = error;
285                    }
286
287                    Event::Token {
288                        source_token: token,
289                        token_kind,
290                        state: TokenState::new(state, masked),
291                    }
292                }
293                event::Event::Directive { directive, masked } => {
294                    if !masked {
295                        match directive.kind() {
296                            DirectiveKind::Version(version) => {
297                                self.type_table.current_version = version.number;
298                            }
299                            DirectiveKind::Extension(extension) => {
300                                if !self.type_table.handle_extension(extension) {
301                                    self.pending_error = Some(
302                                        Error::builder()
303                                            .pos(directive.text_range())
304                                            .resolve_file(self.inner.location())
305                                            .finish(ErrorKind::unsupported_ext(
306                                                extension.name.clone(),
307                                                directive.text_range(),
308                                                self.inner.location(),
309                                            )),
310                                    );
311                                }
312                            }
313                            _ => {}
314                        }
315                    }
316
317                    Event::Directive { directive, masked }
318                }
319            }),
320            Err(err) => Err(err),
321        })
322    }
323}
324
325impl<'r, E, I: Iterator<Item = Result<event::Event, E>> + LocatedIterator> FusedIterator
326    for Tokenizer<'r, I>
327{
328}
329
330impl<'r, I: FileIdResolver> FileIdResolver for Tokenizer<'r, I> {
331    fn resolve(&self, file_id: FileId) -> Option<&std::path::Path> {
332        self.inner.resolve(file_id)
333    }
334}
335
336#[cfg(test)]
337mod tests {
338    use lang_util::FileId;
339    use rowan::NodeOrToken;
340
341    use crate::{processor::event::DirectiveKind, types::Token};
342
343    use super::{Event, MaybeToken};
344
345    #[test]
346    fn test_float_constant() {
347        use super::Token::*;
348
349        fn parse(src: &str) -> Vec<super::Token> {
350            crate::processor::str::process(src, crate::processor::ProcessorState::default())
351                .tokenize(100, false, &crate::exts::DEFAULT_REGISTRY)
352                .filter_map(|evt| evt.as_token().map(|(_, kind, _)| kind.clone()))
353                .collect()
354        }
355
356        assert_eq!(parse("1e-34"), &[FLOAT_CONST(1E-34)]);
357        assert_eq!(parse("1e-34f"), &[FLOAT_CONST(1E-34)]);
358        assert_eq!(parse("1E-34f"), &[FLOAT_CONST(1E-34)]);
359        assert_eq!(parse("1e-34F"), &[FLOAT_CONST(1E-34)]);
360        assert_eq!(parse("1E-34F"), &[FLOAT_CONST(1E-34)]);
361    }
362
363    #[test]
364    /// Ensure that we can extract #(...) for glsl-lang-quote. This is not part of the spec so this
365    /// explains why we need to re-examine tokens using Tokenizer::tokenize_single.
366    ///
367    /// The default behavior is spec-compliant (#( is an invalid preprocessor directive), but we
368    /// can still get this alternative behavior where #(...) is interpreted as a quoting
369    /// interpolation.
370    fn test_hash_ident() {
371        use super::Token::*;
372
373        let inputs = [
374            "#(ident) = hello",
375            "# (ident) = hello",
376            "#(ident.clone()) = hello",
377            "# (ident.clone()) = hello",
378        ];
379        let outputs: [&[Token]; 4] = [
380            &[
381                HASH,
382                LPAREN,
383                IDENT("ident".into()),
384                RPAREN,
385                WS,
386                EQUAL,
387                WS,
388                IDENT("hello".into()),
389            ],
390            &[
391                HASH,
392                WS,
393                LPAREN,
394                IDENT("ident".into()),
395                RPAREN,
396                WS,
397                EQUAL,
398                WS,
399                IDENT("hello".into()),
400            ],
401            &[
402                HASH,
403                LPAREN,
404                IDENT("ident".into()),
405                PERIOD,
406                IDENT("clone".into()),
407                LPAREN,
408                RPAREN,
409                RPAREN,
410                WS,
411                EQUAL,
412                WS,
413                IDENT("hello".into()),
414            ],
415            &[
416                HASH,
417                WS,
418                LPAREN,
419                IDENT("ident".into()),
420                PERIOD,
421                IDENT("clone".into()),
422                LPAREN,
423                RPAREN,
424                RPAREN,
425                WS,
426                EQUAL,
427                WS,
428                IDENT("hello".into()),
429            ],
430        ];
431
432        for (input, output) in inputs.iter().zip(outputs.iter()) {
433            let mut tokens = Vec::new();
434
435            let mut tokenizer =
436                crate::processor::str::process(input, crate::processor::ProcessorState::default())
437                    .tokenize(100, false, &crate::exts::DEFAULT_REGISTRY);
438
439            #[allow(clippy::while_let_on_iterator)]
440            while let Some(result) = tokenizer.next() {
441                if let Ok(event) = result {
442                    match event {
443                        Event::Token { token_kind, .. } => {
444                            tokens.push(token_kind);
445                        }
446                        Event::Directive { directive, .. }
447                            if matches!(directive.kind(), DirectiveKind::Invalid(_)) =>
448                        {
449                            // Extract the tokens from the directive parse tree
450                            tokens.extend(
451                                directive
452                                    .node
453                                    .descendants_with_tokens()
454                                    .filter_map(NodeOrToken::into_token)
455                                    .map(|token| {
456                                        tokenizer.tokenize_single(&(&token, FileId::default())).0
457                                    }),
458                            );
459                        }
460                        _ => {}
461                    }
462                }
463            }
464
465            assert_eq!(&tokens, output);
466        }
467    }
468}
glsl_lang_pp/last.rs

glsl_lang_pp/
last.rs
