lang_util_derive/
token.rs

1use std::collections::HashMap;
2
3use darling::{FromDeriveInput, FromField, FromMeta, FromVariant};
4use proc_macro2::{Span, TokenStream};
5use quote::{format_ident, quote, quote_spanned};
6use syn::{parse_macro_input, spanned::Spanned, DeriveInput, LitStr, Token};
7
8#[derive(FromField)]
9struct TokenVariantField {
10    ident: Option<syn::Ident>,
11}
12
13struct TokenDisplay {
14    format: String,
15    args: Vec<String>,
16}
17
18impl FromMeta for TokenDisplay {
19    fn from_string(value: &str) -> darling::Result<Self> {
20        Ok(Self {
21            format: value.to_owned(),
22            args: vec![],
23        })
24    }
25
26    fn from_list(items: &[darling::ast::NestedMeta]) -> darling::Result<Self> {
27        Ok(Self {
28            format: String::from_nested_meta(
29                items
30                    .first()
31                    .ok_or_else(|| darling::Error::custom("missing format string"))?,
32            )?,
33            args: items
34                .iter()
35                .skip(1)
36                .map(String::from_nested_meta)
37                .collect::<Result<Vec<_>, _>>()?,
38        })
39    }
40}
41
42struct TokenAttr {
43    token: String,
44}
45
46impl FromMeta for TokenAttr {
47    fn from_list(items: &[darling::ast::NestedMeta]) -> darling::Result<Self> {
48        Ok(Self {
49            token: String::from_nested_meta(
50                items
51                    .first()
52                    .ok_or_else(|| darling::Error::custom("missing token literal"))?,
53            )?,
54        })
55    }
56}
57
58enum AsParser {
59    Path(syn::Path),
60    RawString(String),
61}
62
63impl FromMeta for AsParser {
64    fn from_string(value: &str) -> darling::Result<Self> {
65        Ok(Self::RawString(value.to_owned()))
66    }
67
68    fn from_nested_meta(item: &darling::ast::NestedMeta) -> darling::Result<Self> {
69        match item {
70            darling::ast::NestedMeta::Meta(syn::Meta::Path(p)) => Ok(Self::Path(p.to_owned())),
71            _ => Err(darling::Error::unsupported_format("meta")),
72        }
73    }
74
75    fn from_list(items: &[darling::ast::NestedMeta]) -> darling::Result<Self> {
76        if let Some(item) = items.first() {
77            return Self::from_nested_meta(item);
78        }
79
80        Err(darling::Error::unsupported_format("list"))
81    }
82}
83
84#[derive(FromVariant)]
85#[darling(attributes(lang_util), forward_attrs(token))]
86struct TokenVariant {
87    ident: syn::Ident,
88    fields: darling::ast::Fields<TokenVariantField>,
89    attrs: Vec<syn::Attribute>,
90    #[darling(default)]
91    display: Option<TokenDisplay>,
92    #[darling(default, rename = "parser")]
93    as_parser: Option<AsParser>,
94    #[darling(default, rename = "token")]
95    fallback_token: Option<String>,
96    #[darling(multiple, rename = "kind")]
97    kinds: Vec<String>,
98}
99
100struct Token<'s> {
101    base_ident: &'s syn::Ident,
102    variant: &'s TokenVariant,
103    token: TokenAttrTy<'s>,
104    as_parser: Result<String, AsParserError>,
105}
106
107impl<'s> Token<'s> {
108    fn empty_variant_header(&self) -> TokenStream {
109        let variant_name = &self.variant.ident;
110        let variant_header = match &self.variant.fields.style {
111            darling::ast::Style::Tuple => {
112                let fields = self
113                    .variant
114                    .fields
115                    .fields
116                    .iter()
117                    .map(|_| Token![_](Span::call_site()));
118
119                quote! { #variant_name ( #(#fields),* ) }
120            }
121            darling::ast::Style::Struct => {
122                quote! { .. }
123            }
124            darling::ast::Style::Unit => quote! { #variant_name },
125        };
126
127        let base_ident = self.base_ident;
128        quote_spanned! { self.variant.ident.span() => #base_ident :: #variant_header }
129    }
130
131    fn variant_name_arm(&self) -> TokenStream {
132        let variant_header = self.empty_variant_header();
133
134        let body = {
135            let value = &self.variant.ident.to_string();
136            quote_spanned! { self.variant.ident.span() => #value }
137        };
138
139        quote_spanned! {
140            self.variant.ident.span() =>
141                #variant_header => #body
142        }
143    }
144
145    fn parser_token_body(&self) -> TokenStream {
146        match &self.as_parser {
147            Ok(value) => quote_spanned! { self.variant.ident.span() => #value },
148            Err(error) => {
149                let error = error.to_string();
150                quote_spanned! {
151                    self.variant.ident.span() =>
152                        compile_error!(#error)
153                }
154            }
155        }
156    }
157
158    fn parser_token_arm(&self) -> TokenStream {
159        let variant_header = self.empty_variant_header();
160        let body = self.parser_token_body();
161
162        quote_spanned! {
163            self.variant.ident.span() =>
164                #variant_header => #body
165        }
166    }
167
168    fn kinds_body(&self) -> TokenStream {
169        if self.variant.kinds.is_empty() {
170            if let Some((token, attr)) = &self.token {
171                match token {
172                    Ok(value) => {
173                        let value = &value.token;
174                        quote_spanned! { self.variant.ident.span() => &[#value] }
175                    }
176                    Err(error) => {
177                        let s = format!("invalid token attribute: {}", error);
178                        quote_spanned! {
179                            attr.span() =>
180                                compile_error!(#s)
181                        }
182                    }
183                }
184            } else {
185                quote_spanned! {
186                    self.variant.ident.span() =>
187                        compile_error!("cannot determine token kind for this token")
188                }
189            }
190        } else {
191            let kinds = &self.variant.kinds;
192
193            if let Some((Ok(token), _)) = &self.token {
194                let value = &token.token;
195                quote_spanned! { self.variant.ident.span() => &[#(#kinds),*, #value] }
196            } else {
197                quote_spanned! { self.variant.ident.span() => &[#(#kinds),*] }
198            }
199        }
200    }
201
202    fn kinds_arm(&self) -> TokenStream {
203        let variant_header = self.empty_variant_header();
204        let body = self.kinds_body();
205
206        quote_spanned! {
207            self.variant.ident.span() =>
208                #variant_header => #body
209        }
210    }
211
212    fn get_prefixed_fmt(&self, base_fmt: &str) -> String {
213        if self.variant.kinds.is_empty() {
214            format!("`{}`", base_fmt)
215        } else {
216            format!("{} `{}`", self.variant.kinds.last().unwrap(), base_fmt)
217        }
218    }
219
220    fn display_arm_body(&self, declared_fields: &[syn::Ident]) -> TokenStream {
221        match self.variant.fields.style {
222            darling::ast::Style::Tuple => {
223                if let Some(display) = &self.variant.display {
224                    let fmt = &display.format;
225
226                    // TODO: Better than this ugly format string parsing?
227                    let args = if display.args.is_empty() {
228                        if declared_fields.len() == 1 && display.format.contains("{}") {
229                            let f = &declared_fields[0];
230                            vec![quote_spanned! { self.variant.ident.span() => #f }]
231                        } else {
232                            declared_fields
233                                .iter()
234                                .enumerate()
235                                .filter_map(|(i, df)| {
236                                    let ph = format!("_{}", i);
237
238                                    if fmt.contains(&ph) {
239                                        let ph = format_ident!("{}", ph);
240                                        Some(quote_spanned! { self.variant.ident.span() => #ph = #df })
241                                    } else {
242                                        None
243                                    }
244                                })
245                                .collect()
246                        }
247                    } else {
248                        let mut repl = Vec::new();
249                        for (i, df) in declared_fields.iter().enumerate() {
250                            repl.push((format!("_{}", i), df.to_string()));
251                        }
252
253                        display
254                            .args
255                            .iter()
256                            .map(|arg| {
257                                let mut arg = arg.clone();
258                                for (src, dst) in &repl {
259                                    arg = arg.replace(src, dst);
260                                }
261
262                                syn::parse_str(&arg).expect("parsing error")
263                            })
264                            .collect()
265                    };
266
267                    let fmt = self.get_prefixed_fmt(fmt);
268                    return quote_spanned! {
269                        self.variant.ident.span() =>
270                            write!(f, #fmt, #(#args),*)
271                    };
272                } else {
273                    // No display attribute, check if we can have a sensible default
274                    if declared_fields.len() == 1 {
275                        let v1 = &declared_fields[0];
276                        let fmt = self.get_prefixed_fmt("{}");
277                        return quote_spanned! {
278                            self.variant.ident.span() =>
279                                write!(f, #fmt, #v1)
280                        };
281                    }
282                }
283            }
284            darling::ast::Style::Struct => {
285                return quote_spanned! {
286                    self.variant.ident.span() =>
287                        compile_error!("struct variants are not supported yet by derive(Token)")
288                };
289            }
290            darling::ast::Style::Unit => {
291                // Unit struct, is there a token impl?
292                if let Some((token, attr)) = &self.token {
293                    return match token {
294                        Ok(value) => {
295                            let value = &value.token;
296                            let fmt = self.get_prefixed_fmt("{}");
297                            quote_spanned! { self.variant.ident.span() => write!(f, #fmt, #value) }
298                        }
299                        Err(error) => {
300                            let s = format!("invalid token attribute: {}", error);
301                            quote_spanned! {
302                                attr.span() =>
303                                    compile_error!(#s)
304                            }
305                        }
306                    };
307                } else if let Some(display) = &self.variant.display {
308                    let value = &display.format;
309                    let fmt = self.get_prefixed_fmt("{}");
310                    return quote_spanned! { self.variant.ident.span() => write!(f, #fmt, #value) };
311                }
312            }
313        }
314
315        quote_spanned! {
316            self.variant.ident.span() =>
317                compile_error!("missing token or lang_util(display(...)) attributes")
318        }
319    }
320
321    fn display_arm(&self) -> TokenStream {
322        // Create arm header
323        let variant_name = &self.variant.ident;
324        let (variant_header, declared_fields) = match &self.variant.fields.style {
325            darling::ast::Style::Tuple => {
326                let fields: Vec<_> = self
327                    .variant
328                    .fields
329                    .fields
330                    .iter()
331                    .enumerate()
332                    .map(|(i, _field)| format_ident!("value_{}", i))
333                    .collect();
334
335                (
336                    quote_spanned! { self.variant.ident.span() => #variant_name ( #(#fields),* ) },
337                    fields,
338                )
339            }
340            darling::ast::Style::Struct => {
341                let fields: Vec<_> = self
342                    .variant
343                    .fields
344                    .fields
345                    .iter()
346                    .map(|field| field.ident.as_ref().unwrap().clone())
347                    .collect();
348
349                (
350                    quote_spanned! { self.variant.ident.span() => #variant_name { #(#fields),* } },
351                    fields,
352                )
353            }
354            darling::ast::Style::Unit => (quote! { #variant_name }, vec![]),
355        };
356
357        // Create arm body
358        let body = self.display_arm_body(&declared_fields);
359
360        let base_ident = self.base_ident;
361        quote_spanned! {
362            self.variant.ident.span() =>
363                #base_ident :: #variant_header => { #body }
364        }
365    }
366
367    fn all_tokens_arm(&self) -> TokenStream {
368        let variant_name = self.variant.ident.to_string();
369        let parser_token = self.parser_token_body();
370        let kinds = self.kinds_body();
371
372        quote_spanned! {
373            self.variant.ident.span() =>
374                ::lang_util::token::TokenDescriptor::new(#variant_name, #parser_token, #kinds)
375        }
376    }
377}
378
379type TokenAttrTy<'s> = Option<(darling::Result<TokenAttr>, proc_macro2::Span)>;
380
381fn parse_token_attr(variant: &TokenVariant) -> TokenAttrTy {
382    // Unit struct, is there a token impl?
383    for attr in variant.attrs.iter() {
384        if attr.path().is_ident("token") {
385            return Some((
386                if let Ok(result) = attr.parse_args::<LitStr>() {
387                    Ok(TokenAttr {
388                        token: result.value(),
389                    })
390                } else {
391                    attr.parse_args()
392                        .map_err(darling::Error::custom)
393                        .and_then(|meta| TokenAttr::from_meta(&meta))
394                },
395                attr.span(),
396            ));
397        }
398    }
399
400    // Check for the fallback attr
401    if let Some(fallback) = &variant.fallback_token {
402        return Some((
403            Ok(TokenAttr {
404                token: fallback.clone(),
405            }),
406            proc_macro2::Span::call_site(),
407        ));
408    }
409
410    None
411}
412
413#[derive(Debug)]
414enum AsParserError {
415    MissingDisplayImpl,
416    InvalidAs,
417    InvalidTokenAttribute(String),
418    NoTokenOrAs,
419}
420
421impl std::fmt::Display for AsParserError {
422    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
423        match self {
424            Self::MissingDisplayImpl => write!(f, "in lang_util attribute, `parser = display` specified but not display implementation provided"),
425            Self::InvalidAs => write!(f, "invalid `parser` value, expected display or a literal string"),
426            Self::InvalidTokenAttribute(error) => write!(f, "invalid token attribute: {}", error),
427            Self::NoTokenOrAs => write!(f, "missing token or lang_util(parser = \"...\") attributes"),
428        }
429    }
430}
431
432fn parse_as_parser(variant: &TokenVariant, token: &TokenAttrTy) -> Result<String, AsParserError> {
433    if let Some(as_parser) = &variant.as_parser {
434        match as_parser {
435            AsParser::Path(path) => {
436                if path.get_ident().map(|id| id == "display").unwrap_or(false) {
437                    if let Some(display) = &variant.display {
438                        Ok(format!("\"{}\"", &display.format))
439                    } else {
440                        Err(AsParserError::MissingDisplayImpl)
441                    }
442                } else {
443                    Err(AsParserError::InvalidAs)
444                }
445            }
446            AsParser::RawString(s) => Ok(s.to_owned()),
447        }
448    } else if let Some((token, _)) = &token {
449        match token {
450            Ok(value) => {
451                // Tokens would be wrapped with quotes
452                Ok(format!("\"{}\"", &value.token))
453            }
454            Err(error) => Err(AsParserError::InvalidTokenAttribute(error.to_string())),
455        }
456    } else {
457        Err(AsParserError::NoTokenOrAs)
458    }
459}
460
461impl<'s> From<(&'s syn::Ident, &'s TokenVariant)> for Token<'s> {
462    fn from((base_ident, variant): (&'s syn::Ident, &'s TokenVariant)) -> Self {
463        let token = parse_token_attr(variant);
464        let as_parser = parse_as_parser(variant, &token);
465
466        Self {
467            base_ident,
468            variant,
469            token,
470            as_parser,
471        }
472    }
473}
474
475#[derive(FromDeriveInput)]
476#[darling(attributes(lang_util))]
477struct TokenOpts {
478    ident: syn::Ident,
479    generics: syn::Generics,
480    data: darling::ast::Data<TokenVariant, ()>,
481}
482
483fn display_impl(
484    base_ident: &syn::Ident,
485    enum_name: &TokenStream,
486    variants: &[Token],
487) -> TokenStream {
488    let arms = variants.iter().map(Token::display_arm);
489
490    quote_spanned! {
491        base_ident.span() =>
492            impl ::std::fmt::Display for #enum_name {
493                fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
494                    match self {
495                        #(#arms),*
496                    }
497                }
498            }
499    }
500}
501
502fn token_impl(base_ident: &syn::Ident, enum_name: &TokenStream, variants: &[Token]) -> TokenStream {
503    let variant_name_arms = variants.iter().map(Token::variant_name_arm);
504    let parser_token_arms = variants.iter().map(Token::parser_token_arm);
505    let kinds_arms = variants.iter().map(Token::kinds_arm);
506    let all_tokens_arms = variants.iter().map(Token::all_tokens_arm);
507
508    let id = format_ident!("__{}_TOKENS", base_ident.to_string().to_uppercase());
509    let cnt = variants.len();
510
511    quote_spanned! {
512        base_ident.span() =>
513            static #id: [::lang_util::token::TokenDescriptor; #cnt] = [
514                #(#all_tokens_arms),*
515            ];
516
517            impl ::lang_util::token::Token for #enum_name {
518                fn variant_name(&self) -> &'static str {
519                    match self {
520                        #(#variant_name_arms),*
521                    }
522                }
523
524                fn parser_token(&self) -> &'static str {
525                    match self {
526                        #(#parser_token_arms),*
527                    }
528                }
529
530                fn kinds(&self) -> &'static [&'static str] {
531                    match self {
532                        #(#kinds_arms),*
533                    }
534                }
535
536                fn all_tokens() -> &'static [::lang_util::token::TokenDescriptor] {
537                    &#id
538                }
539            }
540    }
541}
542
543pub(crate) fn token(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
544    let opts = {
545        // Parse the input tokens into a syntax tree
546        let input = parse_macro_input!(input as DeriveInput);
547        // Find out enum-level options
548        match TokenOpts::from_derive_input(&input) {
549            Ok(res) => res,
550            Err(err) => return err.write_errors().into(),
551        }
552    };
553
554    let base_ident = &opts.ident;
555
556    // Extract enum fields
557    let fields = match &opts.data {
558        darling::ast::Data::Enum(fields) => fields,
559        darling::ast::Data::Struct(_) => {
560            return proc_macro::TokenStream::from(quote_spanned! {
561                opts.ident.span() =>
562                    compile_error!("only enums can be used as tokens with derive(Token)")
563            })
564        }
565    };
566
567    // Create the Token structs
568    let fields: Vec<Token> = fields
569        .iter()
570        .map(|variant| (base_ident, variant).into())
571        .collect();
572
573    // All declarations
574    let mut decls = Vec::new();
575
576    // Check unicity of the as declarations
577    let mut seen: HashMap<_, &Token> = HashMap::new();
578    for decl in &fields {
579        if let Ok(as_parser) = &decl.as_parser {
580            if let Some(previous) = seen.get(as_parser) {
581                let s = format!(
582                    "`{}` parser token already declared by variant {}",
583                    as_parser, previous.variant.ident
584                );
585                decls.push(quote_spanned! { decl.variant.ident.span() => compile_error!(#s); });
586            } else {
587                seen.insert(as_parser, decl);
588            }
589        }
590    }
591
592    // Compute enum name
593    let enum_name = {
594        // Add anonymous lifetimes as needed
595        let lifetimes: Vec<_> = opts.generics.lifetimes().map(|_| quote! { '_ }).collect();
596
597        if lifetimes.is_empty() {
598            quote! { #base_ident }
599        } else {
600            quote! { #base_ident<#(#lifetimes),*> }
601        }
602    };
603
604    decls.push(display_impl(base_ident, &enum_name, &fields));
605    decls.push(token_impl(base_ident, &enum_name, &fields));
606
607    proc_macro::TokenStream::from(quote_spanned! {
608        opts.ident.span() =>
609            #(#decls)*
610    })
611}
lang_util_derive/token.rs

lang_util_derive/
token.rs
