Enum Token

#[repr(u16)]
pub enum Token {
    IDENT_KW = 1,
    DEFINED = 2,
    DIGITS = 3,
    PERIOD = 4,
    PLUS = 5,
    DASH = 6,
    SLASH = 7,
    ASTERISK = 8,
    PERCENT = 9,
    LANGLE = 10,
    RANGLE = 11,
    LBRACKET = 12,
    RBRACKET = 13,
    LPAREN = 14,
    RPAREN = 15,
    LBRACE = 16,
    RBRACE = 17,
    CARET = 18,
    BAR = 19,
    AMPERSAND = 20,
    TILDE = 21,
    EQUAL = 22,
    BANG = 23,
    COLON = 24,
    SEMICOLON = 25,
    COMMA = 26,
    QUESTION = 27,
    HASH = 28,
    QUOTE_STRING = 29,
    ANGLE_STRING = 30,
    BACKSLASH = 31,
    WS = 32,
    NEWLINE = 33,
    COMMENT = 34,
    LINECONT = 35,
    ERROR = 36,
}

Variants

IDENT_KW = 1

Identifier or keyword

DEFINED = 2

defined preprocessor keyword

DIGITS = 3

Digit sequence

PERIOD = 4

.

PLUS = 5

DASH = 6

SLASH = 7

/

ASTERISK = 8

PERCENT = 9

%

LANGLE = 10

<

RANGLE = 11

LBRACKET = 12

[

RBRACKET = 13

]

LPAREN = 14

(

RPAREN = 15

)

LBRACE = 16

{

RBRACE = 17

}

CARET = 18

^

BAR = 19

|

AMPERSAND = 20

&

TILDE = 21

~

EQUAL = 22

=

BANG = 23

!

COLON = 24

:

SEMICOLON = 25

;

COMMA = 26

,

QUESTION = 27

?

HASH = 28

QUOTE_STRING = 29

“string”

ANGLE_STRING = 30

BACKSLASH = 31

\

WS = 32

Whitespace

NEWLINE = 33

Newline

COMMENT = 34

Comment (single-line or multi-line)

LINECONT = 35

Line continuation (required for tracking exact offsets)

ERROR = 36

Invalid token

Implementations

impl Token

pub fn is_trivia(&self) -> bool

pub fn from_punct(raw: &str) -> Self

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl From<Token> for Token

fn from(pre: Token) -> Self

Converts to this type from the input type.

impl PartialEq for Token

fn eq(&self, other: &Token) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Copy for Token

impl Eq for Token

impl StructuralPartialEq for Token