//! Token definitions for Fermented `UnrealScript`. //! //! These are the tokens consumed by the parser and derived from [`RawToken`]s. use super::{BraceKind, raw_lexer::RawToken}; /// Tokens consumed by the Fermented `UnrealScript` parser. #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Token { ExecDirective, Keyword(Keyword), // Primaries FloatLiteral, IntegerLiteral, StringLiteral, NameLiteral, Identifier, // Operations Exponentiation, Increment, Decrement, Not, BitwiseNot, Multiply, Divide, Modulo, Plus, Minus, ConcatSpace, Concat, LeftShift, LogicalRightShift, RightShift, Less, LessEqual, Greater, GreaterEqual, Equal, NotEqual, ApproximatelyEqual, BitwiseAnd, BitwiseOr, BitwiseXor, LogicalAnd, LogicalXor, LogicalOr, Assign, MultiplyAssign, DivideAssign, ModuloAssign, PlusAssign, MinusAssign, ConcatAssign, ConcatSpaceAssign, // Delimiters LeftParenthesis, RightParenthesis, LeftBrace, CppBlock, RightBrace, LeftBracket, RightBracket, Semicolon, Comma, Period, Colon, Hash, Question, // Trivia LineComment, BlockComment, Newline, Whitespace, // Technical - for representing a very wrong sequence of characters Error, } impl From for Token { #![allow(clippy::too_many_lines)] fn from(token: RawToken) -> Self { match token { // Non-trivial conversions RawToken::Brace(BraceKind::Normal) => Self::LeftBrace, RawToken::Brace(BraceKind::CppBlock) => Self::CppBlock, // Keyword conversions RawToken::CppText => Self::Keyword(Keyword::CppText), RawToken::CppStruct => Self::Keyword(Keyword::CppStruct), RawToken::Class => Self::Keyword(Keyword::Class), RawToken::Struct => Self::Keyword(Keyword::Struct), RawToken::Enum => Self::Keyword(Keyword::Enum), RawToken::State => Self::Keyword(Keyword::State), RawToken::Auto => Self::Keyword(Keyword::Auto), RawToken::Function => Self::Keyword(Keyword::Function), RawToken::Event => Self::Keyword(Keyword::Event), RawToken::Delegate => Self::Keyword(Keyword::Delegate), RawToken::Var => Self::Keyword(Keyword::Var), RawToken::Local => Self::Keyword(Keyword::Local), RawToken::Extends => Self::Keyword(Keyword::Extends), RawToken::DependsOn => Self::Keyword(Keyword::DependsOn), RawToken::Private => Self::Keyword(Keyword::Private), RawToken::Protected => Self::Keyword(Keyword::Protected), RawToken::Public => Self::Keyword(Keyword::Public), RawToken::Const => Self::Keyword(Keyword::Const), RawToken::Static => Self::Keyword(Keyword::Static), RawToken::Native => Self::Keyword(Keyword::Native), RawToken::Abstract => Self::Keyword(Keyword::Abstract), RawToken::Deprecated => Self::Keyword(Keyword::Deprecated), RawToken::SafeReplace => Self::Keyword(Keyword::SafeReplace), RawToken::ExportStructs => Self::Keyword(Keyword::ExportStructs), RawToken::Input => Self::Keyword(Keyword::Input), RawToken::Final => Self::Keyword(Keyword::Final), RawToken::Default => Self::Keyword(Keyword::Default), RawToken::DefaultProperties => Self::Keyword(Keyword::DefaultProperties), RawToken::Object => Self::Keyword(Keyword::Object), RawToken::Begin => Self::Keyword(Keyword::Begin), RawToken::End => Self::Keyword(Keyword::End), RawToken::Optional => Self::Keyword(Keyword::Optional), RawToken::Config => Self::Keyword(Keyword::Config), RawToken::PerObjectConfig => Self::Keyword(Keyword::PerObjectConfig), RawToken::GlobalConfig => Self::Keyword(Keyword::GlobalConfig), RawToken::CollapseCategories => Self::Keyword(Keyword::CollapseCategories), RawToken::DontCollapseCategories => Self::Keyword(Keyword::DontCollapseCategories), RawToken::HideCategories => Self::Keyword(Keyword::HideCategories), RawToken::ShowCategories => Self::Keyword(Keyword::ShowCategories), RawToken::Localized => Self::Keyword(Keyword::Localized), RawToken::Placeable => Self::Keyword(Keyword::Placeable), RawToken::NotPlaceable => Self::Keyword(Keyword::NotPlaceable), RawToken::Instanced => Self::Keyword(Keyword::Instanced), RawToken::EditConst => Self::Keyword(Keyword::EditConst), RawToken::EditConstArray => Self::Keyword(Keyword::EditConstArray), RawToken::EditInline => Self::Keyword(Keyword::EditInline), RawToken::EditInlineUse => Self::Keyword(Keyword::EditInlineUse), RawToken::EditInlineNew => Self::Keyword(Keyword::EditInlineNew), RawToken::NotEditInlineNew => Self::Keyword(Keyword::NotEditInlineNew), RawToken::EdFindable => Self::Keyword(Keyword::EdFindable), RawToken::EditInlineNotify => Self::Keyword(Keyword::EditInlineNotify), RawToken::ParseConfig => Self::Keyword(Keyword::ParseConfig), RawToken::Automated => Self::Keyword(Keyword::Automated), RawToken::DynamicRecompile => Self::Keyword(Keyword::DynamicRecompile), RawToken::Transient => Self::Keyword(Keyword::Transient), RawToken::Long => Self::Keyword(Keyword::Long), RawToken::Operator => Self::Keyword(Keyword::Operator), RawToken::PreOperator => Self::Keyword(Keyword::PreOperator), RawToken::PostOperator => Self::Keyword(Keyword::PostOperator), RawToken::Simulated => Self::Keyword(Keyword::Simulated), RawToken::Exec => Self::Keyword(Keyword::Exec), RawToken::Latent => Self::Keyword(Keyword::Latent), RawToken::Iterator => Self::Keyword(Keyword::Iterator), RawToken::Out => Self::Keyword(Keyword::Out), RawToken::Skip => Self::Keyword(Keyword::Skip), RawToken::Singular => Self::Keyword(Keyword::Singular), RawToken::Coerce => Self::Keyword(Keyword::Coerce), RawToken::Assert => Self::Keyword(Keyword::Assert), RawToken::Ignores => Self::Keyword(Keyword::Ignores), RawToken::Within => Self::Keyword(Keyword::Within), RawToken::Init => Self::Keyword(Keyword::Init), RawToken::Export => Self::Keyword(Keyword::Export), RawToken::NoExport => Self::Keyword(Keyword::NoExport), RawToken::HideDropdown => Self::Keyword(Keyword::HideDropdown), RawToken::Travel => Self::Keyword(Keyword::Travel), RawToken::Cache => Self::Keyword(Keyword::Cache), RawToken::CacheExempt => Self::Keyword(Keyword::CacheExempt), RawToken::Reliable => Self::Keyword(Keyword::Reliable), RawToken::Unreliable => Self::Keyword(Keyword::Unreliable), RawToken::Replication => Self::Keyword(Keyword::Replication), RawToken::NativeReplication => Self::Keyword(Keyword::NativeReplication), RawToken::Goto => Self::Keyword(Keyword::Goto), RawToken::If => Self::Keyword(Keyword::If), RawToken::Else => Self::Keyword(Keyword::Else), RawToken::Switch => Self::Keyword(Keyword::Switch), RawToken::Case => Self::Keyword(Keyword::Case), RawToken::For => Self::Keyword(Keyword::For), RawToken::ForEach => Self::Keyword(Keyword::ForEach), RawToken::While => Self::Keyword(Keyword::While), RawToken::Do => Self::Keyword(Keyword::Do), RawToken::Until => Self::Keyword(Keyword::Until), RawToken::Break => Self::Keyword(Keyword::Break), RawToken::Continue => Self::Keyword(Keyword::Continue), RawToken::Return => Self::Keyword(Keyword::Return), RawToken::Int => Self::Keyword(Keyword::Int), RawToken::Float => Self::Keyword(Keyword::Float), RawToken::Bool => Self::Keyword(Keyword::Bool), RawToken::Byte => Self::Keyword(Keyword::Byte), RawToken::String => Self::Keyword(Keyword::String), RawToken::Array => Self::Keyword(Keyword::Array), RawToken::Name => Self::Keyword(Keyword::Name), RawToken::True => Self::Keyword(Keyword::True), RawToken::False => Self::Keyword(Keyword::False), RawToken::None => Self::Keyword(Keyword::None), RawToken::SelfValue => Self::Keyword(Keyword::SelfValue), RawToken::New => Self::Keyword(Keyword::New), RawToken::Dot => Self::Keyword(Keyword::Dot), RawToken::Cross => Self::Keyword(Keyword::Cross), RawToken::ClockwiseFrom => Self::Keyword(Keyword::ClockwiseFrom), // Trivial 1-to-1 conversions. RawToken::ExecDirective => Self::ExecDirective, RawToken::FloatLiteral => Self::FloatLiteral, RawToken::IntegerLiteral => Self::IntegerLiteral, RawToken::StringLiteral => Self::StringLiteral, RawToken::NameLiteral => Self::NameLiteral, RawToken::Identifier => Self::Identifier, RawToken::Exponentiation => Self::Exponentiation, RawToken::Increment => Self::Increment, RawToken::Decrement => Self::Decrement, RawToken::Not => Self::Not, RawToken::BitwiseNot => Self::BitwiseNot, RawToken::Multiply => Self::Multiply, RawToken::Divide => Self::Divide, RawToken::Modulo => Self::Modulo, RawToken::Plus => Self::Plus, RawToken::Minus => Self::Minus, RawToken::ConcatSpace => Self::ConcatSpace, RawToken::Concat => Self::Concat, RawToken::LeftShift => Self::LeftShift, RawToken::LogicalRightShift => Self::LogicalRightShift, RawToken::RightShift => Self::RightShift, RawToken::Less => Self::Less, RawToken::LessEqual => Self::LessEqual, RawToken::Greater => Self::Greater, RawToken::GreaterEqual => Self::GreaterEqual, RawToken::Equal => Self::Equal, RawToken::NotEqual => Self::NotEqual, RawToken::ApproximatelyEqual => Self::ApproximatelyEqual, RawToken::BitwiseAnd => Self::BitwiseAnd, RawToken::BitwiseOr => Self::BitwiseOr, RawToken::BitwiseXor => Self::BitwiseXor, RawToken::LogicalAnd => Self::LogicalAnd, RawToken::LogicalXor => Self::LogicalXor, RawToken::LogicalOr => Self::LogicalOr, RawToken::Assign => Self::Assign, RawToken::MultiplyAssign => Self::MultiplyAssign, RawToken::DivideAssign => Self::DivideAssign, RawToken::ModuloAssign => Self::ModuloAssign, RawToken::PlusAssign => Self::PlusAssign, RawToken::MinusAssign => Self::MinusAssign, RawToken::ConcatAssign => Self::ConcatAssign, RawToken::ConcatSpaceAssign => Self::ConcatSpaceAssign, RawToken::LeftParenthesis => Self::LeftParenthesis, RawToken::RightParenthesis => Self::RightParenthesis, RawToken::RightBrace => Self::RightBrace, RawToken::LeftBracket => Self::LeftBracket, RawToken::RightBracket => Self::RightBracket, RawToken::Semicolon => Self::Semicolon, RawToken::Comma => Self::Comma, RawToken::Period => Self::Period, RawToken::Colon => Self::Colon, RawToken::Hash => Self::Hash, RawToken::Question => Self::Question, RawToken::LineComment => Self::LineComment, RawToken::BlockComment => Self::BlockComment, RawToken::Newline => Self::Newline, RawToken::Whitespace => Self::Whitespace, RawToken::Error => Self::Error, } } } impl Token { /// Returns `true` if this token is a newline. #[must_use] pub const fn is_newline(&self) -> bool { matches!(self, Self::Newline) } /// Returns `true` if this token is trivia whitespace. /// /// Note: comments are **not** considered whitespace. #[must_use] pub const fn is_whitespace(&self) -> bool { matches!(self, Self::Whitespace | Self::Newline) } /// Returns `true` if this token may span multiple physical lines /// (i.e. can contain newline characters). #[must_use] pub const fn can_span_lines(&self) -> bool { matches!(self, Self::BlockComment | Self::CppBlock | Self::Error) } /// Returns `true` if this token can appear in type position /// (either a built-in type keyword or an identifier). #[must_use] pub fn is_valid_type_name(&self) -> bool { let Self::Keyword(keyword) = self else { return *self == Self::Identifier; }; keyword.is_valid_type_name() } /// Returns `true` if this token can be used as an identifier. /// /// This includes [`Token::Identifier`] and certain keywords that /// `UnrealScript` also accepts in identifier position. #[must_use] pub fn is_valid_identifier_name(&self) -> bool { if *self == Self::Identifier { return true; } if let Self::Keyword(keyword) = self { return keyword.is_valid_identifier_name(); } false } /// Returns `true` if this token can be used as function's modifier. #[must_use] pub const fn is_valid_function_modifier(&self) -> bool { let Self::Keyword(keyword) = self else { return false; }; matches!( keyword, Keyword::Final | Keyword::Native | Keyword::Abstract | Keyword::Transient | Keyword::Public | Keyword::Protected | Keyword::Private | Keyword::Static | Keyword::Const | Keyword::Deprecated | Keyword::NoExport | Keyword::Export | Keyword::Simulated | Keyword::Latent | Keyword::Iterator | Keyword::Singular | Keyword::Reliable | Keyword::Unreliable | Keyword::NativeReplication | Keyword::PreOperator | Keyword::Operator | Keyword::PostOperator | Keyword::Config | Keyword::Exec ) } } /// Reserved words of Fermented `UnrealScript`. /// /// These are represented in [`Token`] as [`Token::Keyword`]. #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Keyword { // C++ blocks CppText, CppStruct, // Top-level declaration Class, Struct, Enum, State, Auto, Function, Event, Delegate, Var, Local, // Class modifiers Extends, DependsOn, // Access modifiers Private, Protected, Public, Const, // Meta data / specifiers Static, Native, Abstract, Deprecated, SafeReplace, ExportStructs, Input, Final, Default, DefaultProperties, Object, Begin, End, Optional, Config, PerObjectConfig, GlobalConfig, CollapseCategories, DontCollapseCategories, HideCategories, ShowCategories, Localized, Placeable, NotPlaceable, Instanced, EditConst, EditConstArray, EditInline, EditInlineUse, EditInlineNew, NotEditInlineNew, EdFindable, EditInlineNotify, ParseConfig, Automated, DynamicRecompile, Transient, Long, Operator, PreOperator, PostOperator, Simulated, Exec, Latent, Iterator, Out, Skip, Singular, Coerce, Assert, Ignores, Within, Init, Export, NoExport, HideDropdown, Travel, Cache, CacheExempt, // Replication Reliable, Unreliable, Replication, NativeReplication, // Control flow Goto, If, Else, Switch, Case, For, ForEach, While, Do, Until, Break, Continue, Return, // Built-in types Int, Float, Bool, Byte, String, Array, Name, // Literals True, False, None, SelfValue, New, // Vector math operators Dot, Cross, ClockwiseFrom, } impl Keyword { /// Returns `true` if this keyword can be used as an identifier. #[must_use] pub const fn is_valid_identifier_name(self) -> bool { matches!( self, // Built-in type words usable as identifiers Self::Name | Self::String | Self::Byte | Self::Int | Self::Bool | Self::Float | Self::Array | Self::Delegate // Context keywords we've directly checked | Self::Class | Self::SelfValue | Self::Default | Self::Static | Self::Simulated | Self::Native | Self::Latent | Self::Iterator | Self::Singular | Self::Reliable | Self::Unreliable | Self::Transient | Self::Const | Self::Abstract | Self::New | Self::Extends | Self::Within | Self::Config | Self::Out | Self::Optional | Self::Local | Self::Var | Self::DefaultProperties | Self::PerObjectConfig | Self::Object | Self::Enum | Self::End | Self::Event | Self::Switch | Self::Goto | Self::Cross | Self::CppText | Self::CppStruct | Self::HideCategories | Self::Auto | Self::For | Self::Skip | Self::Placeable | Self::NotPlaceable | Self::Instanced | Self::Function | Self::State | Self::Init | Self::Export | Self::NoExport | Self::Dot | Self::ClockwiseFrom | Self::Assert | Self::ExportStructs | Self::SafeReplace | Self::Input | Self::Travel | Self::Cache | Self::CacheExempt | Self::Long | Self::Continue ) } /// Returns `true` if this keyword can appear in type position. #[must_use] pub const fn is_valid_type_name(self) -> bool { matches!( self, Self::Int | Self::Float | Self::Bool | Self::Byte | Self::String | Self::Array | Self::Name | Self::Object | Self::Function | Self::State | Self::Delegate ) } }