rott/rottlib/src/lexer/token.rs

//! Token definitions for Fermented UnrealScript.
//!
//! These are the tokens consumed by the parser and derived from [`RawToken`]s.

use super::{BraceKind, raw_lexer::RawToken};

/// Tokens consumed by the Fermented UnrealScript parser.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Token {
    ExecDirective,
    Keyword(Keyword),
    // Primaries
    FloatLiteral,
    IntegerLiteral,
    StringLiteral,
    NameLiteral,
    Identifier,
    // Operations
    Exponentiation,
    Increment,
    Decrement,
    Not,
    BitwiseNot,
    Multiply,
    Divide,
    Modulo,
    Plus,
    Minus,
    ConcatSpace,
    Concat,
    LeftShift,
    LogicalRightShift,
    RightShift,
    Less,
    LessEqual,
    Greater,
    GreaterEqual,
    Equal,
    NotEqual,
    ApproximatelyEqual,
    BitwiseAnd,
    BitwiseOr,
    BitwiseXor,
    LogicalAnd,
    LogicalXor,
    LogicalOr,
    Assign,
    MultiplyAssign,
    DivideAssign,
    ModuloAssign,
    PlusAssign,
    MinusAssign,
    ConcatAssign,
    ConcatSpaceAssign,
    // Delimiters
    LeftParenthesis,
    RightParenthesis,
    LeftBrace,
    CppBlock,
    RightBrace,
    LeftBracket,
    RightBracket,
    Semicolon,
    Comma,
    Period,
    Colon,
    Hash,
    Question,
    // Trivia
    LineComment,
    BlockComment,
    Newline,
    Whitespace,
    // Technical - for representing a very wrong sequence of characters
    Error,
}

impl From<RawToken> for Token {
    #![allow(clippy::too_many_lines)]
    fn from(token: RawToken) -> Self {
        match token {
            // Non-trivial conversions
            RawToken::Brace(BraceKind::Normal) => Self::LeftBrace,
            RawToken::Brace(BraceKind::CppBlock) => Self::CppBlock,
            // Keyword conversions
            RawToken::CppText => Self::Keyword(Keyword::CppText),
            RawToken::CppStruct => Self::Keyword(Keyword::CppStruct),
            RawToken::Class => Self::Keyword(Keyword::Class),
            RawToken::Struct => Self::Keyword(Keyword::Struct),
            RawToken::Enum => Self::Keyword(Keyword::Enum),
            RawToken::State => Self::Keyword(Keyword::State),
            RawToken::Auto => Self::Keyword(Keyword::Auto),
            RawToken::Function => Self::Keyword(Keyword::Function),
            RawToken::Event => Self::Keyword(Keyword::Event),
            RawToken::Delegate => Self::Keyword(Keyword::Delegate),
            RawToken::Var => Self::Keyword(Keyword::Var),
            RawToken::Local => Self::Keyword(Keyword::Local),
            RawToken::Extends => Self::Keyword(Keyword::Extends),
            RawToken::DependsOn => Self::Keyword(Keyword::DependsOn),
            RawToken::Private => Self::Keyword(Keyword::Private),
            RawToken::Protected => Self::Keyword(Keyword::Protected),
            RawToken::Public => Self::Keyword(Keyword::Public),
            RawToken::Const => Self::Keyword(Keyword::Const),
            RawToken::Static => Self::Keyword(Keyword::Static),
            RawToken::Native => Self::Keyword(Keyword::Native),
            RawToken::Abstract => Self::Keyword(Keyword::Abstract),
            RawToken::Deprecated => Self::Keyword(Keyword::Deprecated),
            RawToken::SafeReplace => Self::Keyword(Keyword::SafeReplace),
            RawToken::ExportStructs => Self::Keyword(Keyword::ExportStructs),
            RawToken::Input => Self::Keyword(Keyword::Input),
            RawToken::Final => Self::Keyword(Keyword::Final),
            RawToken::Default => Self::Keyword(Keyword::Default),
            RawToken::DefaultProperties => Self::Keyword(Keyword::DefaultProperties),
            RawToken::Object => Self::Keyword(Keyword::Object),
            RawToken::Begin => Self::Keyword(Keyword::Begin),
            RawToken::End => Self::Keyword(Keyword::End),
            RawToken::Optional => Self::Keyword(Keyword::Optional),
            RawToken::Config => Self::Keyword(Keyword::Config),
            RawToken::PerObjectConfig => Self::Keyword(Keyword::PerObjectConfig),
            RawToken::GlobalConfig => Self::Keyword(Keyword::GlobalConfig),
            RawToken::CollapseCategories => Self::Keyword(Keyword::CollapseCategories),
            RawToken::DontCollapseCategories => Self::Keyword(Keyword::DontCollapseCategories),
            RawToken::HideCategories => Self::Keyword(Keyword::HideCategories),
            RawToken::ShowCategories => Self::Keyword(Keyword::ShowCategories),
            RawToken::Localized => Self::Keyword(Keyword::Localized),
            RawToken::Placeable => Self::Keyword(Keyword::Placeable),
            RawToken::NotPlaceable => Self::Keyword(Keyword::NotPlaceable),
            RawToken::Instanced => Self::Keyword(Keyword::Instanced),
            RawToken::EditConst => Self::Keyword(Keyword::EditConst),
            RawToken::EditConstArray => Self::Keyword(Keyword::EditConstArray),
            RawToken::EditInline => Self::Keyword(Keyword::EditInline),
            RawToken::EditInlineUse => Self::Keyword(Keyword::EditInlineUse),
            RawToken::EditInlineNew => Self::Keyword(Keyword::EditInlineNew),
            RawToken::NotEditInlineNew => Self::Keyword(Keyword::NotEditInlineNew),
            RawToken::EdFindable => Self::Keyword(Keyword::EdFindable),
            RawToken::EditInlineNotify => Self::Keyword(Keyword::EditInlineNotify),
            RawToken::ParseConfig => Self::Keyword(Keyword::ParseConfig),
            RawToken::Automated => Self::Keyword(Keyword::Automated),
            RawToken::DynamicRecompile => Self::Keyword(Keyword::DynamicRecompile),
            RawToken::Transient => Self::Keyword(Keyword::Transient),
            RawToken::Long => Self::Keyword(Keyword::Long),
            RawToken::Operator => Self::Keyword(Keyword::Operator),
            RawToken::PreOperator => Self::Keyword(Keyword::PreOperator),
            RawToken::PostOperator => Self::Keyword(Keyword::PostOperator),
            RawToken::Simulated => Self::Keyword(Keyword::Simulated),
            RawToken::Exec => Self::Keyword(Keyword::Exec),
            RawToken::Latent => Self::Keyword(Keyword::Latent),
            RawToken::Iterator => Self::Keyword(Keyword::Iterator),
            RawToken::Out => Self::Keyword(Keyword::Out),
            RawToken::Skip => Self::Keyword(Keyword::Skip),
            RawToken::Singular => Self::Keyword(Keyword::Singular),
            RawToken::Coerce => Self::Keyword(Keyword::Coerce),
            RawToken::Assert => Self::Keyword(Keyword::Assert),
            RawToken::Ignores => Self::Keyword(Keyword::Ignores),
            RawToken::Within => Self::Keyword(Keyword::Within),
            RawToken::Init => Self::Keyword(Keyword::Init),
            RawToken::Export => Self::Keyword(Keyword::Export),
            RawToken::NoExport => Self::Keyword(Keyword::NoExport),
            RawToken::HideDropdown => Self::Keyword(Keyword::HideDropdown),
            RawToken::Travel => Self::Keyword(Keyword::Travel),
            RawToken::Cache => Self::Keyword(Keyword::Cache),
            RawToken::CacheExempt => Self::Keyword(Keyword::CacheExempt),
            RawToken::Reliable => Self::Keyword(Keyword::Reliable),
            RawToken::Unreliable => Self::Keyword(Keyword::Unreliable),
            RawToken::Replication => Self::Keyword(Keyword::Replication),
            RawToken::NativeReplication => Self::Keyword(Keyword::NativeReplication),
            RawToken::Goto => Self::Keyword(Keyword::Goto),
            RawToken::If => Self::Keyword(Keyword::If),
            RawToken::Else => Self::Keyword(Keyword::Else),
            RawToken::Switch => Self::Keyword(Keyword::Switch),
            RawToken::Case => Self::Keyword(Keyword::Case),
            RawToken::For => Self::Keyword(Keyword::For),
            RawToken::ForEach => Self::Keyword(Keyword::ForEach),
            RawToken::While => Self::Keyword(Keyword::While),
            RawToken::Do => Self::Keyword(Keyword::Do),
            RawToken::Until => Self::Keyword(Keyword::Until),
            RawToken::Break => Self::Keyword(Keyword::Break),
            RawToken::Continue => Self::Keyword(Keyword::Continue),
            RawToken::Return => Self::Keyword(Keyword::Return),
            RawToken::Int => Self::Keyword(Keyword::Int),
            RawToken::Float => Self::Keyword(Keyword::Float),
            RawToken::Bool => Self::Keyword(Keyword::Bool),
            RawToken::Byte => Self::Keyword(Keyword::Byte),
            RawToken::String => Self::Keyword(Keyword::String),
            RawToken::Array => Self::Keyword(Keyword::Array),
            RawToken::Name => Self::Keyword(Keyword::Name),
            RawToken::True => Self::Keyword(Keyword::True),
            RawToken::False => Self::Keyword(Keyword::False),
            RawToken::None => Self::Keyword(Keyword::None),
            RawToken::SelfValue => Self::Keyword(Keyword::SelfValue),
            RawToken::New => Self::Keyword(Keyword::New),
            RawToken::Dot => Self::Keyword(Keyword::Dot),
            RawToken::Cross => Self::Keyword(Keyword::Cross),
            RawToken::ClockwiseFrom => Self::Keyword(Keyword::ClockwiseFrom),
            // Trivial 1-to-1 conversions.
            RawToken::ExecDirective => Self::ExecDirective,
            RawToken::FloatLiteral => Self::FloatLiteral,
            RawToken::IntegerLiteral => Self::IntegerLiteral,
            RawToken::StringLiteral => Self::StringLiteral,
            RawToken::NameLiteral => Self::NameLiteral,
            RawToken::Identifier => Self::Identifier,
            RawToken::Exponentiation => Self::Exponentiation,
            RawToken::Increment => Self::Increment,
            RawToken::Decrement => Self::Decrement,
            RawToken::Not => Self::Not,
            RawToken::BitwiseNot => Self::BitwiseNot,
            RawToken::Multiply => Self::Multiply,
            RawToken::Divide => Self::Divide,
            RawToken::Modulo => Self::Modulo,
            RawToken::Plus => Self::Plus,
            RawToken::Minus => Self::Minus,
            RawToken::ConcatSpace => Self::ConcatSpace,
            RawToken::Concat => Self::Concat,
            RawToken::LeftShift => Self::LeftShift,
            RawToken::LogicalRightShift => Self::LogicalRightShift,
            RawToken::RightShift => Self::RightShift,
            RawToken::Less => Self::Less,
            RawToken::LessEqual => Self::LessEqual,
            RawToken::Greater => Self::Greater,
            RawToken::GreaterEqual => Self::GreaterEqual,
            RawToken::Equal => Self::Equal,
            RawToken::NotEqual => Self::NotEqual,
            RawToken::ApproximatelyEqual => Self::ApproximatelyEqual,
            RawToken::BitwiseAnd => Self::BitwiseAnd,
            RawToken::BitwiseOr => Self::BitwiseOr,
            RawToken::BitwiseXor => Self::BitwiseXor,
            RawToken::LogicalAnd => Self::LogicalAnd,
            RawToken::LogicalXor => Self::LogicalXor,
            RawToken::LogicalOr => Self::LogicalOr,
            RawToken::Assign => Self::Assign,
            RawToken::MultiplyAssign => Self::MultiplyAssign,
            RawToken::DivideAssign => Self::DivideAssign,
            RawToken::ModuloAssign => Self::ModuloAssign,
            RawToken::PlusAssign => Self::PlusAssign,
            RawToken::MinusAssign => Self::MinusAssign,
            RawToken::ConcatAssign => Self::ConcatAssign,
            RawToken::ConcatSpaceAssign => Self::ConcatSpaceAssign,
            RawToken::LeftParenthesis => Self::LeftParenthesis,
            RawToken::RightParenthesis => Self::RightParenthesis,
            RawToken::RightBrace => Self::RightBrace,
            RawToken::LeftBracket => Self::LeftBracket,
            RawToken::RightBracket => Self::RightBracket,
            RawToken::Semicolon => Self::Semicolon,
            RawToken::Comma => Self::Comma,
            RawToken::Period => Self::Period,
            RawToken::Colon => Self::Colon,
            RawToken::Hash => Self::Hash,
            RawToken::Question => Self::Question,
            RawToken::LineComment => Self::LineComment,
            RawToken::BlockComment => Self::BlockComment,
            RawToken::Newline => Self::Newline,
            RawToken::Whitespace => Self::Whitespace,
            RawToken::Error => Self::Error,
        }
    }
}

impl Token {
    /// Returns `true` if this token is a newline.
    #[must_use]
    pub const fn is_newline(&self) -> bool {
        matches!(self, Self::Newline)
    }

    /// Returns `true` if this token is trivia whitespace.
    ///
    /// Note: comments are **not** considered whitespace.
    #[must_use]
    pub const fn is_whitespace(&self) -> bool {
        matches!(self, Self::Whitespace | Self::Newline)
    }

    /// Returns `true` if this token may span multiple physical lines
    /// (i.e. can contain newline characters).
    #[must_use]
    pub const fn can_span_lines(&self) -> bool {
        matches!(self, Self::BlockComment | Self::CppBlock | Self::Error)
    }

    /// Returns `true` if this token can appear in type position
    /// (either a built-in type keyword or an identifier).
    #[must_use]
    pub fn is_valid_type_name(&self) -> bool {
        let Self::Keyword(keyword) = self else {
            return *self == Self::Identifier;
        };
        keyword.is_valid_type_name()
    }

    /// Returns `true` if this token can be used as an identifier.
    ///
    /// This includes [`Token::Identifier`] and certain keywords that
    /// `UnrealScript` also accepts in identifier position.
    #[must_use]
    pub fn is_valid_identifier_name(&self) -> bool {
        if *self == Self::Identifier {
            return true;
        }
        if let Self::Keyword(keyword) = self {
            return keyword.is_valid_identifier_name();
        }
        false
    }

    /// Returns `true` if this token can be used as function's modifier.
    #[must_use]
    pub const fn is_valid_function_modifier(&self) -> bool {
        let Self::Keyword(keyword) = self else {
            return false;
        };
        matches!(
            keyword,
            Keyword::Final
                | Keyword::Native
                | Keyword::Abstract
                | Keyword::Transient
                | Keyword::Public
                | Keyword::Protected
                | Keyword::Private
                | Keyword::Static
                | Keyword::Const
                | Keyword::Deprecated
                | Keyword::NoExport
                | Keyword::Export
                | Keyword::Simulated
                | Keyword::Latent
                | Keyword::Iterator
                | Keyword::Singular
                | Keyword::Reliable
                | Keyword::Unreliable
                | Keyword::NativeReplication
                | Keyword::PreOperator
                | Keyword::Operator
                | Keyword::PostOperator
                | Keyword::Config
                | Keyword::Exec
        )
    }

    /// Returns `true` if this token is definitely not a valid first token of an
    /// expression.
    ///
    /// This is a conservative recovery predicate:
    /// - `true` means expression parsing should not be attempted at this token;
    /// - `false` means the token might start an expression, or that the normal
    ///   expression parser should report the more specific error.
    #[must_use]
    pub const fn is_definitely_not_expression_start(self) -> bool {
        match self {
            Self::Keyword(keyword) => keyword.is_definitely_not_expression_start(),

            // Closing delimiters / separators.
            Self::RightParenthesis
            | Self::RightBrace
            | Self::RightBracket
            | Self::Semicolon
            | Self::Comma
            | Self::Colon
            | Self::Question

            // Tokens that only continue a previous expression.
            | Self::Period

            // Infix / postfix / assignment operators.
            | Self::Exponentiation
            | Self::Multiply
            | Self::Divide
            | Self::Modulo
            | Self::ConcatSpace
            | Self::Concat
            | Self::LeftShift
            | Self::LogicalRightShift
            | Self::RightShift
            | Self::Less
            | Self::LessEqual
            | Self::Greater
            | Self::GreaterEqual
            | Self::Equal
            | Self::NotEqual
            | Self::ApproximatelyEqual
            | Self::BitwiseAnd
            | Self::BitwiseOr
            | Self::BitwiseXor
            | Self::LogicalAnd
            | Self::LogicalXor
            | Self::LogicalOr
            | Self::Assign
            | Self::MultiplyAssign
            | Self::DivideAssign
            | Self::ModuloAssign
            | Self::PlusAssign
            | Self::MinusAssign
            | Self::ConcatAssign
            | Self::ConcatSpaceAssign

            // Non-expression trivia / technical tokens.
            | Self::ExecDirective
            | Self::CppBlock
            | Self::Hash
            | Self::LineComment
            | Self::BlockComment
            | Self::Newline
            | Self::Whitespace
            | Self::Error => true,

            _ => false,
        }
    }
}

/// Reserved words of Fermented UnrealScript.
///
/// These are represented in [`Token`] as [`Token::Keyword`].
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Keyword {
    // C++ blocks
    CppText,
    CppStruct,
    // Top-level declaration
    Class,
    Struct,
    Enum,
    State,
    Auto,
    Function,
    Event,
    Delegate,
    Var,
    Local,
    // Class modifiers
    Extends,
    DependsOn,
    // Access modifiers
    Private,
    Protected,
    Public,
    Const,
    // Meta data / specifiers
    Static,
    Native,
    Abstract,
    Deprecated,
    SafeReplace,
    ExportStructs,
    Input,
    Final,
    Default,
    DefaultProperties,
    Object,
    Begin,
    End,
    Optional,
    Config,
    PerObjectConfig,
    GlobalConfig,
    CollapseCategories,
    DontCollapseCategories,
    HideCategories,
    ShowCategories,
    Localized,
    Placeable,
    NotPlaceable,
    Instanced,
    EditConst,
    EditConstArray,
    EditInline,
    EditInlineUse,
    EditInlineNew,
    NotEditInlineNew,
    EdFindable,
    EditInlineNotify,
    ParseConfig,
    Automated,
    DynamicRecompile,
    Transient,
    Long,
    Operator,
    PreOperator,
    PostOperator,
    Simulated,
    Exec,
    Latent,
    Iterator,
    Out,
    Skip,
    Singular,
    Coerce,
    Assert,
    Ignores,
    Within,
    Init,
    Export,
    NoExport,
    HideDropdown,
    Travel,
    Cache,
    CacheExempt,
    // Replication
    Reliable,
    Unreliable,
    Replication,
    NativeReplication,
    // Control flow
    Goto,
    If,
    Else,
    Switch,
    Case,
    For,
    ForEach,
    While,
    Do,
    Until,
    Break,
    Continue,
    Return,
    // Built-in types
    Int,
    Float,
    Bool,
    Byte,
    String,
    Array,
    Name,
    // Literals
    True,
    False,
    None,
    SelfValue,
    New,
    // Vector math operators
    Dot,
    Cross,
    ClockwiseFrom,
}

impl Keyword {
    /// Returns `true` if this keyword can be used as an identifier.
    #[must_use]
    pub const fn is_valid_identifier_name(self) -> bool {
        matches!(
            self,
            // Built-in type words usable as identifiers
            Self::Name
                | Self::String
                | Self::Byte
                | Self::Int
                | Self::Bool
                | Self::Float
                | Self::Array
                | Self::Delegate
                // Context keywords we've directly checked
                | Self::Class
                | Self::SelfValue
                | Self::Default
                | Self::Static
                | Self::Simulated
                | Self::Native
                | Self::Latent
                | Self::Iterator
                | Self::Singular
                | Self::Reliable
                | Self::Unreliable
                | Self::Transient
                | Self::Const
                | Self::Abstract
                | Self::New
                | Self::Extends
                | Self::Within
                | Self::Config
                | Self::Out
                | Self::Optional
                | Self::Local
                | Self::Var
                | Self::DefaultProperties
                | Self::PerObjectConfig
                | Self::Object
                | Self::Enum
                | Self::End
                | Self::Event
                | Self::Switch
                | Self::Goto
                | Self::Cross
                | Self::CppText
                | Self::CppStruct
                | Self::HideCategories
                | Self::Auto
                | Self::For
                | Self::Skip
                | Self::Placeable
                | Self::NotPlaceable
                | Self::Instanced
                | Self::Function
                | Self::State
                | Self::Init
                | Self::Export
                | Self::NoExport
                | Self::Dot
                | Self::ClockwiseFrom
                | Self::Assert
                | Self::ExportStructs
                | Self::SafeReplace
                | Self::Input
                | Self::Travel
                | Self::Cache
                | Self::CacheExempt
                | Self::Long
                | Self::Continue
        )
    }

    /// Returns `true` if this keyword can appear in type position.
    #[must_use]
    pub const fn is_valid_type_name(self) -> bool {
        matches!(
            self,
            Self::Int
                | Self::Float
                | Self::Bool
                | Self::Byte
                | Self::String
                | Self::Array
                | Self::Name
                | Self::Object
                | Self::Function
                | Self::State
                | Self::Delegate
        )
    }

    /// Returns `true` if this keyword is definitely not a valid first token of
    /// an expression.
    #[must_use]
    pub const fn is_definitely_not_expression_start(self) -> bool {
        matches!(self, Self::Else | Self::Case | Self::Until)
    }
}