rott/rottlib/src/lexer/token.rs

//! Token definitions for Fermented `UnrealScript`.
//!
//! These are the tokens consumed by the parser and derived from [`RawToken`]s.

use super::{BraceKind, raw_lexer::RawToken};

/// Tokens consumed by the Fermented `UnrealScript` parser.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Token {
    ExecDirective,
    Keyword(Keyword),
    // Primaries
    FloatLiteral,
    IntegerLiteral,
    StringLiteral,
    NameLiteral,
    Identifier,
    // Operations
    Exponentiation,
    Increment,
    Decrement,
    Not,
    BitwiseNot,
    Multiply,
    Divide,
    Modulo,
    Plus,
    Minus,
    ConcatSpace,
    Concat,
    LeftShift,
    LogicalRightShift,
    RightShift,
    Less,
    LessEqual,
    Greater,
    GreaterEqual,
    Equal,
    NotEqual,
    ApproximatelyEqual,
    BitwiseAnd,
    BitwiseOr,
    BitwiseXor,
    LogicalAnd,
    LogicalXor,
    LogicalOr,
    Assign,
    MultiplyAssign,
    DivideAssign,
    ModuloAssign,
    PlusAssign,
    MinusAssign,
    ConcatAssign,
    ConcatSpaceAssign,
    // Delimiters
    LeftParenthesis,
    RightParenthesis,
    LeftBrace,
    CppBlock,
    RightBrace,
    LeftBracket,
    RightBracket,
    Semicolon,
    Comma,
    Period,
    Colon,
    Hash,
    Question,
    // Trivia
    LineComment,
    BlockComment,
    Newline,
    Whitespace,
    // Technical - for representing a very wrong sequence of characters
    Error,
}

impl From<RawToken> for Token {
    #![allow(clippy::too_many_lines)]
    fn from(token: RawToken) -> Self {
        match token {
            // Non-trivial conversions
            RawToken::Brace(BraceKind::Normal) => Self::LeftBrace,
            RawToken::Brace(BraceKind::CppBlock) => Self::CppBlock,
            // Keyword conversions
            RawToken::CppText => Self::Keyword(Keyword::CppText),
            RawToken::CppStruct => Self::Keyword(Keyword::CppStruct),
            RawToken::Class => Self::Keyword(Keyword::Class),
            RawToken::Struct => Self::Keyword(Keyword::Struct),
            RawToken::Enum => Self::Keyword(Keyword::Enum),
            RawToken::State => Self::Keyword(Keyword::State),
            RawToken::Auto => Self::Keyword(Keyword::Auto),
            RawToken::Function => Self::Keyword(Keyword::Function),
            RawToken::Event => Self::Keyword(Keyword::Event),
            RawToken::Delegate => Self::Keyword(Keyword::Delegate),
            RawToken::Var => Self::Keyword(Keyword::Var),
            RawToken::Local => Self::Keyword(Keyword::Local),
            RawToken::Extends => Self::Keyword(Keyword::Extends),
            RawToken::DependsOn => Self::Keyword(Keyword::DependsOn),
            RawToken::Private => Self::Keyword(Keyword::Private),
            RawToken::Protected => Self::Keyword(Keyword::Protected),
            RawToken::Public => Self::Keyword(Keyword::Public),
            RawToken::Const => Self::Keyword(Keyword::Const),
            RawToken::Static => Self::Keyword(Keyword::Static),
            RawToken::Native => Self::Keyword(Keyword::Native),
            RawToken::Abstract => Self::Keyword(Keyword::Abstract),
            RawToken::Deprecated => Self::Keyword(Keyword::Deprecated),
            RawToken::SafeReplace => Self::Keyword(Keyword::SafeReplace),
            RawToken::ExportStructs => Self::Keyword(Keyword::ExportStructs),
            RawToken::Input => Self::Keyword(Keyword::Input),
            RawToken::Final => Self::Keyword(Keyword::Final),
            RawToken::Default => Self::Keyword(Keyword::Default),
            RawToken::DefaultProperties => Self::Keyword(Keyword::DefaultProperties),
            RawToken::Object => Self::Keyword(Keyword::Object),
            RawToken::Begin => Self::Keyword(Keyword::Begin),
            RawToken::End => Self::Keyword(Keyword::End),
            RawToken::Optional => Self::Keyword(Keyword::Optional),
            RawToken::Config => Self::Keyword(Keyword::Config),
            RawToken::PerObjectConfig => Self::Keyword(Keyword::PerObjectConfig),
            RawToken::GlobalConfig => Self::Keyword(Keyword::GlobalConfig),
            RawToken::CollapseCategories => Self::Keyword(Keyword::CollapseCategories),
            RawToken::DontCollapseCategories => Self::Keyword(Keyword::DontCollapseCategories),
            RawToken::HideCategories => Self::Keyword(Keyword::HideCategories),
            RawToken::ShowCategories => Self::Keyword(Keyword::ShowCategories),
            RawToken::Localized => Self::Keyword(Keyword::Localized),
            RawToken::Placeable => Self::Keyword(Keyword::Placeable),
            RawToken::NotPlaceable => Self::Keyword(Keyword::NotPlaceable),
            RawToken::Instanced => Self::Keyword(Keyword::Instanced),
            RawToken::EditConst => Self::Keyword(Keyword::EditConst),
            RawToken::EditConstArray => Self::Keyword(Keyword::EditConstArray),
            RawToken::EditInline => Self::Keyword(Keyword::EditInline),
            RawToken::EditInlineUse => Self::Keyword(Keyword::EditInlineUse),
            RawToken::EditInlineNew => Self::Keyword(Keyword::EditInlineNew),
            RawToken::NotEditInlineNew => Self::Keyword(Keyword::NotEditInlineNew),
            RawToken::EdFindable => Self::Keyword(Keyword::EdFindable),
            RawToken::EditInlineNotify => Self::Keyword(Keyword::EditInlineNotify),
            RawToken::ParseConfig => Self::Keyword(Keyword::ParseConfig),
            RawToken::Automated => Self::Keyword(Keyword::Automated),
            RawToken::DynamicRecompile => Self::Keyword(Keyword::DynamicRecompile),
            RawToken::Transient => Self::Keyword(Keyword::Transient),
            RawToken::Long => Self::Keyword(Keyword::Long),
            RawToken::Operator => Self::Keyword(Keyword::Operator),
            RawToken::PreOperator => Self::Keyword(Keyword::PreOperator),
            RawToken::PostOperator => Self::Keyword(Keyword::PostOperator),
            RawToken::Simulated => Self::Keyword(Keyword::Simulated),
            RawToken::Exec => Self::Keyword(Keyword::Exec),
            RawToken::Latent => Self::Keyword(Keyword::Latent),
            RawToken::Iterator => Self::Keyword(Keyword::Iterator),
            RawToken::Out => Self::Keyword(Keyword::Out),
            RawToken::Skip => Self::Keyword(Keyword::Skip),
            RawToken::Singular => Self::Keyword(Keyword::Singular),
            RawToken::Coerce => Self::Keyword(Keyword::Coerce),
            RawToken::Assert => Self::Keyword(Keyword::Assert),
            RawToken::Ignores => Self::Keyword(Keyword::Ignores),
            RawToken::Within => Self::Keyword(Keyword::Within),
            RawToken::Init => Self::Keyword(Keyword::Init),
            RawToken::Export => Self::Keyword(Keyword::Export),
            RawToken::NoExport => Self::Keyword(Keyword::NoExport),
            RawToken::HideDropdown => Self::Keyword(Keyword::HideDropdown),
            RawToken::Travel => Self::Keyword(Keyword::Travel),
            RawToken::Cache => Self::Keyword(Keyword::Cache),
            RawToken::CacheExempt => Self::Keyword(Keyword::CacheExempt),
            RawToken::Reliable => Self::Keyword(Keyword::Reliable),
            RawToken::Unreliable => Self::Keyword(Keyword::Unreliable),
            RawToken::Replication => Self::Keyword(Keyword::Replication),
            RawToken::NativeReplication => Self::Keyword(Keyword::NativeReplication),
            RawToken::Goto => Self::Keyword(Keyword::Goto),
            RawToken::If => Self::Keyword(Keyword::If),
            RawToken::Else => Self::Keyword(Keyword::Else),
            RawToken::Switch => Self::Keyword(Keyword::Switch),
            RawToken::Case => Self::Keyword(Keyword::Case),
            RawToken::For => Self::Keyword(Keyword::For),
            RawToken::ForEach => Self::Keyword(Keyword::ForEach),
            RawToken::While => Self::Keyword(Keyword::While),
            RawToken::Do => Self::Keyword(Keyword::Do),
            RawToken::Until => Self::Keyword(Keyword::Until),
            RawToken::Break => Self::Keyword(Keyword::Break),
            RawToken::Continue => Self::Keyword(Keyword::Continue),
            RawToken::Return => Self::Keyword(Keyword::Return),
            RawToken::Int => Self::Keyword(Keyword::Int),
            RawToken::Float => Self::Keyword(Keyword::Float),
            RawToken::Bool => Self::Keyword(Keyword::Bool),
            RawToken::Byte => Self::Keyword(Keyword::Byte),
            RawToken::String => Self::Keyword(Keyword::String),
            RawToken::Array => Self::Keyword(Keyword::Array),
            RawToken::Name => Self::Keyword(Keyword::Name),
            RawToken::True => Self::Keyword(Keyword::True),
            RawToken::False => Self::Keyword(Keyword::False),
            RawToken::None => Self::Keyword(Keyword::None),
            RawToken::SelfValue => Self::Keyword(Keyword::SelfValue),
            RawToken::New => Self::Keyword(Keyword::New),
            RawToken::Dot => Self::Keyword(Keyword::Dot),
            RawToken::Cross => Self::Keyword(Keyword::Cross),
            RawToken::ClockwiseFrom => Self::Keyword(Keyword::ClockwiseFrom),
            // Trivial 1-to-1 conversions.
            RawToken::ExecDirective => Self::ExecDirective,
            RawToken::FloatLiteral => Self::FloatLiteral,
            RawToken::IntegerLiteral => Self::IntegerLiteral,
            RawToken::StringLiteral => Self::StringLiteral,
            RawToken::NameLiteral => Self::NameLiteral,
            RawToken::Identifier => Self::Identifier,
            RawToken::Exponentiation => Self::Exponentiation,
            RawToken::Increment => Self::Increment,
            RawToken::Decrement => Self::Decrement,
            RawToken::Not => Self::Not,
            RawToken::BitwiseNot => Self::BitwiseNot,
            RawToken::Multiply => Self::Multiply,
            RawToken::Divide => Self::Divide,
            RawToken::Modulo => Self::Modulo,
            RawToken::Plus => Self::Plus,
            RawToken::Minus => Self::Minus,
            RawToken::ConcatSpace => Self::ConcatSpace,
            RawToken::Concat => Self::Concat,
            RawToken::LeftShift => Self::LeftShift,
            RawToken::LogicalRightShift => Self::LogicalRightShift,
            RawToken::RightShift => Self::RightShift,
            RawToken::Less => Self::Less,
            RawToken::LessEqual => Self::LessEqual,
            RawToken::Greater => Self::Greater,
            RawToken::GreaterEqual => Self::GreaterEqual,
            RawToken::Equal => Self::Equal,
            RawToken::NotEqual => Self::NotEqual,
            RawToken::ApproximatelyEqual => Self::ApproximatelyEqual,
            RawToken::BitwiseAnd => Self::BitwiseAnd,
            RawToken::BitwiseOr => Self::BitwiseOr,
            RawToken::BitwiseXor => Self::BitwiseXor,
            RawToken::LogicalAnd => Self::LogicalAnd,
            RawToken::LogicalXor => Self::LogicalXor,
            RawToken::LogicalOr => Self::LogicalOr,
            RawToken::Assign => Self::Assign,
            RawToken::MultiplyAssign => Self::MultiplyAssign,
            RawToken::DivideAssign => Self::DivideAssign,
            RawToken::ModuloAssign => Self::ModuloAssign,
            RawToken::PlusAssign => Self::PlusAssign,
            RawToken::MinusAssign => Self::MinusAssign,
            RawToken::ConcatAssign => Self::ConcatAssign,
            RawToken::ConcatSpaceAssign => Self::ConcatSpaceAssign,
            RawToken::LeftParenthesis => Self::LeftParenthesis,
            RawToken::RightParenthesis => Self::RightParenthesis,
            RawToken::RightBrace => Self::RightBrace,
            RawToken::LeftBracket => Self::LeftBracket,
            RawToken::RightBracket => Self::RightBracket,
            RawToken::Semicolon => Self::Semicolon,
            RawToken::Comma => Self::Comma,
            RawToken::Period => Self::Period,
            RawToken::Colon => Self::Colon,
            RawToken::Hash => Self::Hash,
            RawToken::Question => Self::Question,
            RawToken::LineComment => Self::LineComment,
            RawToken::BlockComment => Self::BlockComment,
            RawToken::Newline => Self::Newline,
            RawToken::Whitespace => Self::Whitespace,
            RawToken::Error => Self::Error,
        }
    }
}

impl Token {
    /// Returns `true` if this token is a newline.
    #[must_use]
    pub const fn is_newline(&self) -> bool {
        matches!(self, Self::Newline)
    }

    /// Returns `true` if this token is trivia whitespace.
    ///
    /// Note: comments are **not** considered whitespace.
    #[must_use]
    pub const fn is_whitespace(&self) -> bool {
        matches!(self, Self::Whitespace | Self::Newline)
    }

    /// Returns `true` if this token may span multiple physical lines
    /// (i.e. can contain newline characters).
    #[must_use]
    pub const fn can_span_lines(&self) -> bool {
        matches!(self, Self::BlockComment | Self::CppBlock | Self::Error)
    }

    /// Returns `true` if this token can appear in type position
    /// (either a built-in type keyword or an identifier).
    #[must_use]
    pub fn is_valid_type_name(&self) -> bool {
        let Self::Keyword(keyword) = self else {
            return *self == Self::Identifier;
        };
        keyword.is_valid_type_name()
    }

    /// Returns `true` if this token can be used as an identifier.
    ///
    /// This includes [`Token::Identifier`] and certain keywords that
    /// `UnrealScript` also accepts in identifier position.
    #[must_use]
    pub fn is_valid_identifier_name(&self) -> bool {
        if *self == Self::Identifier {
            return true;
        }
        if let Self::Keyword(keyword) = self {
            return keyword.is_valid_identifier_name();
        }
        false
    }

    /// Returns `true` if this token can be used as function's modifier.
    #[must_use]
    pub const fn is_valid_function_modifier(&self) -> bool {
        let Self::Keyword(keyword) = self else {
            return false;
        };
        matches!(
            keyword,
            Keyword::Final
                | Keyword::Native
                | Keyword::Abstract
                | Keyword::Transient
                | Keyword::Public
                | Keyword::Protected
                | Keyword::Private
                | Keyword::Static
                | Keyword::Const
                | Keyword::Deprecated
                | Keyword::NoExport
                | Keyword::Export
                | Keyword::Simulated
                | Keyword::Latent
                | Keyword::Iterator
                | Keyword::Singular
                | Keyword::Reliable
                | Keyword::Unreliable
                | Keyword::NativeReplication
                | Keyword::PreOperator
                | Keyword::Operator
                | Keyword::PostOperator
                | Keyword::Config
                | Keyword::Exec
        )
    }
}

/// Reserved words of Fermented `UnrealScript`.
///
/// These are represented in [`Token`] as [`Token::Keyword`].
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Keyword {
    // C++ blocks
    CppText,
    CppStruct,
    // Top-level declaration
    Class,
    Struct,
    Enum,
    State,
    Auto,
    Function,
    Event,
    Delegate,
    Var,
    Local,
    // Class modifiers
    Extends,
    DependsOn,
    // Access modifiers
    Private,
    Protected,
    Public,
    Const,
    // Meta data / specifiers
    Static,
    Native,
    Abstract,
    Deprecated,
    SafeReplace,
    ExportStructs,
    Input,
    Final,
    Default,
    DefaultProperties,
    Object,
    Begin,
    End,
    Optional,
    Config,
    PerObjectConfig,
    GlobalConfig,
    CollapseCategories,
    DontCollapseCategories,
    HideCategories,
    ShowCategories,
    Localized,
    Placeable,
    NotPlaceable,
    Instanced,
    EditConst,
    EditConstArray,
    EditInline,
    EditInlineUse,
    EditInlineNew,
    NotEditInlineNew,
    EdFindable,
    EditInlineNotify,
    ParseConfig,
    Automated,
    DynamicRecompile,
    Transient,
    Long,
    Operator,
    PreOperator,
    PostOperator,
    Simulated,
    Exec,
    Latent,
    Iterator,
    Out,
    Skip,
    Singular,
    Coerce,
    Assert,
    Ignores,
    Within,
    Init,
    Export,
    NoExport,
    HideDropdown,
    Travel,
    Cache,
    CacheExempt,
    // Replication
    Reliable,
    Unreliable,
    Replication,
    NativeReplication,
    // Control flow
    Goto,
    If,
    Else,
    Switch,
    Case,
    For,
    ForEach,
    While,
    Do,
    Until,
    Break,
    Continue,
    Return,
    // Built-in types
    Int,
    Float,
    Bool,
    Byte,
    String,
    Array,
    Name,
    // Literals
    True,
    False,
    None,
    SelfValue,
    New,
    // Vector math operators
    Dot,
    Cross,
    ClockwiseFrom,
}

impl Keyword {
    /// Returns `true` if this keyword can be used as an identifier.
    #[must_use]
    pub const fn is_valid_identifier_name(self) -> bool {
        matches!(
            self,
            // Built-in type words usable as identifiers
            Self::Name
                | Self::String
                | Self::Byte
                | Self::Int
                | Self::Bool
                | Self::Float
                | Self::Array
                | Self::Delegate
                // Context keywords we've directly checked
                | Self::Class
                | Self::SelfValue
                | Self::Default
                | Self::Static
                | Self::Simulated
                | Self::Native
                | Self::Latent
                | Self::Iterator
                | Self::Singular
                | Self::Reliable
                | Self::Unreliable
                | Self::Transient
                | Self::Const
                | Self::Abstract
                | Self::New
                | Self::Extends
                | Self::Within
                | Self::Config
                | Self::Out
                | Self::Optional
                | Self::Local
                | Self::Var
                | Self::DefaultProperties
                | Self::PerObjectConfig
                | Self::Object
                | Self::Enum
                | Self::End
                | Self::Event
                | Self::Switch
                | Self::Goto
                | Self::Cross
                | Self::CppText
                | Self::CppStruct
                | Self::HideCategories
                | Self::Auto
                | Self::For
                | Self::Skip
                | Self::Placeable
                | Self::NotPlaceable
                | Self::Instanced
                | Self::Function
                | Self::State
                | Self::Init
                | Self::Export
                | Self::NoExport
                | Self::Dot
                | Self::ClockwiseFrom
                | Self::Assert
                | Self::ExportStructs
                | Self::SafeReplace
                | Self::Input
                | Self::Travel
                | Self::Cache
                | Self::CacheExempt
                | Self::Long
                | Self::Continue
        )
    }

    /// Returns `true` if this keyword can appear in type position.
    #[must_use]
    pub const fn is_valid_type_name(self) -> bool {
        matches!(
            self,
            Self::Int
                | Self::Float
                | Self::Bool
                | Self::Byte
                | Self::String
                | Self::Array
                | Self::Name
                | Self::Object
                | Self::Function
                | Self::State
                | Self::Delegate
        )
    }
}