NoobsGoHOE/rott
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: NoobsGoHOE:797e5ea1927a2f6b0aae01637809356333968482
Branches Tags
NoobsGoHOE:master
NoobsGoHOE:parsing_reimplementation
..
compare: NoobsGoHOE:f695f8a52e3b97b310628f2f65118a34f6a39146
Branches Tags
NoobsGoHOE:parsing_reimplementation
NoobsGoHOE:master

No commits in common. "797e5ea1927a2f6b0aae01637809356333968482" and "f695f8a52e3b97b310628f2f65118a34f6a39146" have entirely different histories.
797e5ea192 ... f695f8a52e

31 changed files with 452 additions and 2481 deletions
Show Stats Expand all files Collapse all files

226
dev_tests/src/verify_expr.rs
View File

@ -11,136 +11,170 @@ use rottlib::diagnostics::Diagnostic;
use rottlib::lexer::TokenizedFile; use rottlib::lexer::TokenizedFile;
use rottlib::parser::Parser; use rottlib::parser::Parser;
const TEST_CASES: &[(&str, &str)] = &[
// P0057 - VarEditorSpecifierExpected
("files/P0057_01.uc", "var(Category,) int A;\n"),
("files/P0057_02.uc", "var(,) int A;\n"),
(
"files/P0057_03.uc",
"var(\n ,\n ,\n Category\n) int A;\n",
),
(
"files/P0057_04.uc",
"var(\n Category,\n\n\n) int A;\n",
),
// P0058 - VarEditorSpecifierListMissingSeparator
("files/P0058_01.uc", "var(Category DisplayName) int A;\n"),
("files/P0058_02.uc", "var(\"Display\" \"Tooltip\") int A;\n"),
(
"files/P0058_03.uc",
"var(\n Category\n DisplayName\n) int A;\n",
),
(
"files/P0058_04.uc",
"var(\n Category\n\n \"Display Name\",\n Advanced\n) int A;\n",
),
// P0059 - VarEditorSpecifierListExpectedCommaOrClosingParenthesis
("files/P0059_01.uc", "var(Category :) int A;\n"),
("files/P0059_02.uc", "var(\"Display\" *) int A;\n"),
(
"files/P0059_03.uc",
"var(\n Category\n [\n) int A;\n",
),
(
"files/P0059_04.uc",
"var(\n \"Display Name\"\n\n ]\n) int A;\n",
),
// P0060 - VarEditorSpecifierListMissingClosingParenthesis
("files/P0060_01.uc", "var(Category\n"),
(
"files/P0060_02.uc",
"var(\n Category,\n \"Display Name\"\n",
),
(
"files/P0060_03.uc",
"var(\n\n\n",
),
(
"files/P0060_04.uc",
"var(\n Category,\n\n \"Display Name\"\n\n",
),
];
/// Lexer-focused fixtures. /// Lexer-focused fixtures.
/// ///
/// Keep these small: the goal is to inspect lexer diagnostics and delimiter /// Keep these small: the goal is to inspect lexer diagnostics and delimiter
/// recovery behavior, not full parser behavior. /// recovery behavior, not full parser behavior.
/*const TEST_CASES: &[(&str, &str)] = &[ const TEST_CASES: &[(&str, &str)] = &[
// P0061 - TypeSpecifierExpected // P0034 - SwitchMissingBody
("files/P0061_01.uc", "{\n local ;\n}\n"), ("files/P0034_01.uc", "switch(A) local\n"),
("files/P0034_02.uc", "switch\n(A)\nvar"),
("files/P0034_03.uc", "switch(\n A\n)\n"),
("files/P0034_04.uc", "switch\n(\n A\n)\n"),
("files/P0034_05.uc", "switch(A)\ncase 1:\n"),
// P0035 - SwitchTopLevelItemNotCase
("files/P0035_01.uc", "switch(A) {\n Log(\"bad\");\n}\n"),
( (
"files/P0061_02.uc", "files/P0035_02.uc",
"{\n local\n ;\n}\n", "switch\n(A)\n{\n Log(\"bad\");\n Log(\"worse\");\n case 1:\n}\n",
),
("files/P0035_03.uc", "switch(A) {\n 123;\n default:\n}\n"),
(
"files/P0035_04.uc",
"switch\n(\n A\n)\n{\n if (A) {}\n case 1:\n}\n",
), ),
( (
"files/P0061_03.uc", "files/P0035_05.uc",
"{\n local\n [\n ] Values;\n}\n", "switch(A) {\n {\n Log(\"nested\");\n }\n case 1:\n}\n",
),
(
"files/P0061_04.uc",
"{\n local\n",
), ),
// P0062 - NamedTypeInvalidName // P0036 - SwitchCaseMissingColon
("files/P0062_01.uc", "{\n local Foo. ;\n}\n"),
( (
"files/P0062_02.uc", "files/P0036_01.uc",
"{\n local\n Foo.\n ;\n}\n", "switch(A) {\n case 1\n case 2:\n}\n",
), ),
( (
"files/P0062_03.uc", "files/P0036_02.uc",
"{\n local Foo..Bar Value;\n}\n", "switch\n(A)\n{\n case\n 1\n default:\n}\n",
), ),
( (
"files/P0062_04.uc", "files/P0036_03.uc",
"{\n local\n SomePackage\n .\n", "switch(A) {\n case (A)\n case B:\n}\n",
),
(
"files/P0036_04.uc",
"switch\n(\n A\n)\n{\n case\n A + B\n default\n :\n}\n",
),
(
"files/P0036_05.uc",
"switch(A) {\n case Foo.Bar(Baz)\n case Other:\n}\n",
), ),
// P0063 - ArrayTypeMissingOpeningAngleBracket // P0037 - SwitchDefaultMissingColon
("files/P0063_01.uc", "{\n local array Values;\n}\n"),
( (
"files/P0063_02.uc", "files/P0037_01.uc",
"{\n local\n array\n Values;\n}\n", "switch(A) {\n default\n if (A) {}\n}\n",
), ),
("files/P0063_03.uc", "{\n local array[] Values;\n}\n"),
( (
"files/P0063_04.uc", "files/P0037_02.uc",
"{\n local\n array", "switch\n(A)\n{\n default\n while (A) {}\n}\n",
),
(
"files/P0037_03.uc",
"switch(A) {\n default\n for (;;) {}\n}\n",
),
(
"files/P0037_04.uc",
"switch\n(\n A\n)\n{\n default\n switch(B) {\n case 1:\n }\n}\n",
),
(
"files/P0037_05.uc",
"switch(A) {\n default\n case 1:\n}\n",
), ),
// P0064 - ArrayTypeMissingClosingAngleBracket // P0038 - SwitchDuplicateDefault
("files/P0064_01.uc", "{\n local array<int Values;\n}\n"),
( (
"files/P0064_02.uc", "files/P0038_01.uc",
"{\n local\n array<\n int\n Values;\n}\n", "switch(A) {\n default:\n default:\n}\n",
), ),
( (
"files/P0064_03.uc", "files/P0038_02.uc",
"{\n local array<class<Actor> ActorClasses;\n}\n", "switch\n(A)\n{\n default\n :\n default\n :\n}\n",
), ),
( (
"files/P0064_04.uc", "files/P0038_03.uc",
"{\n local\n array<\n string", "switch(A) {\n default:\n default:\n default:\n}\n",
),
(
"files/P0038_04.uc",
"switch\n(\n A\n)\n{\n default:\n Log(\"first\");\n default:\n Log(\"second\");\n}\n",
), ),
// P0065 - ArrayTypeMissingElementType // P0039 - SwitchCasesAfterDefault
("files/P0065_01.uc", "{\n local array<> Values;\n}\n"),
( (
"files/P0065_02.uc", "files/P0039_01.uc",
"{\n local\n array<>\n Values;\n}\n", "switch(A) {\n default:\n case 1:\n}\n",
), ),
( (
"files/P0065_03.uc", "files/P0039_02.uc",
"{\n local array<\n > Values;\n}\n", "switch\n(A)\n{\n default\n :\n case\n 1\n :\n}\n",
), ),
( (
"files/P0065_04.uc", "files/P0039_03.uc",
"{\n local array<array<> > NestedValues;\n}\n", "switch(A) {\n default:\n case 1:\n case 2:\n}\n",
), ),
];*/ (
"files/P0039_04.uc",
"switch\n(\n A\n)\n{\n default:\n case 1:\n Log(\"one\");\n case 2:\n Log(\"two\");\n}\n",
),
(
"files/P0039_05.uc",
"switch(A) {\n default:\n Log(\"done\");\n case 1:\n case 2:\n Log(\"stacked\");\n}\n",
),
// P0040 - SwitchMissingClosingBrace
("files/P0040_01.uc", "switch(A) {\n"),
("files/P0040_02.uc", "switch(A) {\n case 1:\n"),
("files/P0040_03.uc", "switch\n(A)\n{\n default:\n"),
(
"files/P0040_04.uc",
"switch\n(\n A\n)\n{\n case 1:\n case 2:\n",
),
(
"files/P0040_05.uc",
"switch(A) {\n case 1:\n Log(\"body\");\n",
),
// P0041 - SwitchCaseMissingExpression
("files/P0041_01.uc", "switch(A) {\n case:\n}\n"),
("files/P0041_02.uc", "switch\n(A)\n{\n case\n :\n}\n"),
("files/P0041_03.uc", "switch(A) {\n case:\n default:\n}\n"),
(
"files/P0041_04.uc",
"switch\n(\n A\n)\n{\n case\n :\n case 1:\n}\n",
),
(
"files/P0041_05.uc",
"switch(A) {\n case\n :\n default:\n}\n",
),
// P0042 - SwitchCaseExpressionInvalidStart
("files/P0042_01.uc", "switch(A) {\n case *:\n}\n"),
(
"files/P0042_02.uc",
"switch\n(A)\n{\n case\n =\n :\n}\n",
),
("files/P0042_03.uc", "switch(A) {\n case &&:\n default:\n}\n"),
(
"files/P0042_04.uc",
"switch\n(\n A\n)\n{\n case\n .\n :\n case 1:\n}\n",
),
(
"files/P0042_05.uc",
"switch(A) {\n case ]:\n}\n",
),
// Mixed / intentional cascades
(
"files/P0042_cascade_01.uc",
"switch(A) {\n case *\n case 1:\n}\n",
),
(
"files/P0042_cascade_02.uc",
"switch\n(\n A\n)\n{\n case\n =\n default:\n}\n",
),
];
/// If true, also run the parser after tokenization. /// If true, also run the parser after tokenization.
/// ///

6
rottlib/src/ast/mod.rs
View File

@ -93,7 +93,7 @@ pub enum Statement<'src, 'arena> {
// `local int i, j, k` // `local int i, j, k`
LocalVariableDeclaration { LocalVariableDeclaration {
type_spec: TypeSpecifierRef<'src, 'arena>, type_spec: TypeSpecifierRef<'src, 'arena>,
declarators: DeclaratorsList<'src, 'arena>, // CHANGED declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>, // CHANGED
}, },
Label(ArenaString<'arena>), Label(ArenaString<'arena>),
/// Nested function definitions inside blocks or states. /// Nested function definitions inside blocks or states.
@ -192,7 +192,7 @@ pub struct ClassVarDecl<'src, 'arena> {
pub modifiers: ArenaVec<'arena, VarModifier>, pub modifiers: ArenaVec<'arena, VarModifier>,
pub type_spec: TypeSpecifierRef<'src, 'arena>, // Named/InlineEnum/InlineStruct pub type_spec: TypeSpecifierRef<'src, 'arena>, // Named/InlineEnum/InlineStruct
pub declarators: DeclaratorsList<'src, 'arena>, // a, b=expr pub declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>, // a, b=expr
pub span: TokenSpan, pub span: TokenSpan,
} }
pub type ClassVarDeclRef<'src, 'arena> = ArenaNode<'arena, ClassVarDecl<'src, 'arena>>; pub type ClassVarDeclRef<'src, 'arena> = ArenaNode<'arena, ClassVarDecl<'src, 'arena>>;
@ -259,7 +259,7 @@ pub struct StateDecl<'src, 'arena> {
pub modifiers: ArenaVec<'arena, StateModifier>, // auto, simulated pub modifiers: ArenaVec<'arena, StateModifier>, // auto, simulated
pub ignores: Option<ArenaVec<'arena, IdentifierToken>>, // 'ignores Foo, Bar;' pub ignores: Option<ArenaVec<'arena, IdentifierToken>>, // 'ignores Foo, Bar;'
/// Body: ordinary statements plus nested function definitions (see `Statement::Function`). /// Body: ordinary statements plus nested function definitions (see `Statement::Function`).
pub body: StatementList<'src, 'arena>, pub body: ArenaVec<'arena, StatementRef<'src, 'arena>>,
pub span: TokenSpan, pub span: TokenSpan,
} }
pub type StateDeclRef<'src, 'arena> = ArenaNode<'arena, StateDecl<'src, 'arena>>; pub type StateDeclRef<'src, 'arena> = ArenaNode<'arena, StateDecl<'src, 'arena>>;

4
rottlib/src/ast/types.rs
View File

@ -114,7 +114,7 @@ pub struct StructField<'src, 'arena> {
/// Examples: /// Examples:
/// - `var int A;` /// - `var int A;`
/// - `var int A, B[4], C = 10;` /// - `var int A, B[4], C = 10;`
pub declarators: DeclaratorsList<'src, 'arena>, pub declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
/// Optional `var(...)` editor specifiers attached to the field declaration. /// Optional `var(...)` editor specifiers attached to the field declaration.
/// ///
/// Example: /// Example:
@ -167,8 +167,6 @@ pub struct VariableDeclarator<'src, 'arena> {
/// The node span is expected to cover the entire declarator, not only the /// The node span is expected to cover the entire declarator, not only the
/// identifier token. /// identifier token.
pub type VariableDeclaratorRef<'src, 'arena> = ArenaNode<'arena, VariableDeclarator<'src, 'arena>>; pub type VariableDeclaratorRef<'src, 'arena> = ArenaNode<'arena, VariableDeclarator<'src, 'arena>>;
/// Declarations in `var` or `local` variable list.
pub type DeclaratorsList<'src, 'arena> = ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>;
/// One item inside `var(...)` editor specifiers. /// One item inside `var(...)` editor specifiers.
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]

441
rottlib/src/diagnostics/parse_error_diagnostics/declarations.rs
View File

@ -1,441 +0,0 @@
use super::{
Diagnostic, DiagnosticBuilder, FoundAt, found_at, primary_span_with_optional_multiline_context,
should_show_context_label,
};
use crate::lexer::{TokenSpan, TokenizedFile};
use crate::parser::{ParseError, diagnostic_labels};
const LOCAL_KEYWORD: &str = "local_keyword";
const DECLARATION_KEYWORD: &str = "declaration_keyword";
const VARIABLE_TYPE: &str = "variable_type";
const LAST_DECLARATOR: &str = "last_declarator";
const SEPARATOR: &str = "separator";
const OPEN_BRACKET: &str = "open_bracket";
fn related_span(error: &ParseError, key: &str) -> Option<TokenSpan> {
error.related_spans.get(key).copied()
}
fn should_label_keyword(
file: &TokenizedFile<'_>,
keyword_span: Option<TokenSpan>,
context_span: Option<TokenSpan>,
blame_span: TokenSpan,
) -> bool {
let Some(keyword_span) = keyword_span else {
return false;
};
let Some(context_span) = context_span else {
return !file.same_line(keyword_span.start, blame_span.end);
};
!file.same_line(keyword_span.start, context_span.end)
&& !file.same_line(keyword_span.start, blame_span.end)
}
fn expected_before_found(
file: &TokenizedFile<'_>,
blame_span: TokenSpan,
expected: &str,
) -> String {
match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => format!("{expected} before `{token_text}`"),
FoundAt::EndOfFile => format!("{expected} before end of file"),
FoundAt::Unknown => format!("{expected} here"),
}
}
fn expected_variable_name_primary_text(file: &TokenizedFile<'_>, blame_span: TokenSpan) -> String {
expected_before_found(file, blame_span, "expected variable name")
}
/// P0050
pub(super) fn diagnostic_local_variable_declaration_missing_semicolon<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let local_keyword_span = related_span(&error, LOCAL_KEYWORD);
let last_declarator_span = related_span(&error, LAST_DECLARATOR);
let blame_span = error.blame_span;
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token("local") => "expected `;` before this declaration".to_string(),
FoundAt::Token(token_text) => format!("expected `;` before `{token_text}`"),
FoundAt::EndOfFile => "expected `;` before end of file".to_string(),
FoundAt::Unknown => "expected `;` here".to_string(),
};
let mut builder = DiagnosticBuilder::error("missing `;` after local variable declaration");
if should_label_keyword(file, local_keyword_span, last_declarator_span, blame_span) {
if let Some(local_keyword_span) = local_keyword_span {
builder = builder
.secondary_label(local_keyword_span, "local variable declaration starts here");
}
}
if let Some(last_declarator_span) = last_declarator_span {
builder = builder.secondary_label(last_declarator_span, "declaration ends here");
}
builder
.primary_label(blame_span, primary_text)
.code("P0050")
.build()
}
/// P0051
pub(super) fn diagnostic_variable_declarator_list_empty<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let variable_type_span = related_span(&error, VARIABLE_TYPE);
let blame_span = error.blame_span;
let primary_text = expected_variable_name_primary_text(file, blame_span);
let mut builder =
DiagnosticBuilder::error("expected at least one variable name in variable declaration");
if should_label_keyword(
file,
declaration_keyword_span,
variable_type_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(variable_type_span) = variable_type_span {
if should_show_context_label(file, variable_type_span, blame_span) {
builder = builder.secondary_label(
variable_type_span,
"after this type, a variable name was expected",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0051")
.build()
}
/// P0052
pub(super) fn diagnostic_variable_declarator_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let variable_type_span = related_span(&error, VARIABLE_TYPE);
let separator_span = related_span(&error, SEPARATOR);
let blame_span = error.blame_span;
let first_declarator = error.flags.contains("first_declarator");
let empty_declarator_run = error.flags.contains("empty_declarator_run");
let blame_is_multi_token = blame_span.start != blame_span.end;
let title = if first_declarator && empty_declarator_run {
"expected variable name before `,`"
} else if separator_span.is_some() {
"expected variable name after `,`"
} else {
"expected variable name in variable declaration"
};
let primary_text = if empty_declarator_run {
match found_at(file, blame_span.start) {
FoundAt::Token(",") if blame_is_multi_token => {
"empty variable declarators are not allowed".to_string()
}
FoundAt::Token(",") => "unexpected `,`".to_string(),
FoundAt::Token(token_text) => {
format!("expected variable name before `{token_text}`")
}
FoundAt::EndOfFile => "expected variable name before end of file".to_string(),
FoundAt::Unknown => "expected variable name here".to_string(),
}
} else if separator_span.is_some() {
match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => {
format!("expected variable name before `{token_text}`")
}
FoundAt::EndOfFile => "expected variable name before end of file".to_string(),
FoundAt::Unknown => "expected variable name here".to_string(),
}
} else {
expected_variable_name_primary_text(file, blame_span)
};
let context_span = separator_span.or(variable_type_span);
let mut builder = DiagnosticBuilder::error(title);
if should_label_keyword(file, declaration_keyword_span, context_span, blame_span) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(separator_span) = separator_span {
if should_show_context_label(file, separator_span, blame_span) {
builder = builder.secondary_label(
separator_span,
"after this `,`, another variable name was expected",
);
}
} else if let Some(variable_type_span) = variable_type_span {
if should_show_context_label(file, variable_type_span, blame_span) {
builder = builder.secondary_label(
variable_type_span,
"after this type, a variable name was expected",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0052")
.build()
}
/// P0053
pub(super) fn diagnostic_variable_declarator_list_missing_separator<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let last_declarator_span = related_span(&error, LAST_DECLARATOR);
let blame_span = error.blame_span;
let primary_text = expected_before_found(file, blame_span, "expected `,`");
let mut builder = DiagnosticBuilder::error("missing `,` between variable declarators");
if should_label_keyword(
file,
declaration_keyword_span,
last_declarator_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(last_declarator_span) = last_declarator_span {
builder = builder.secondary_label(last_declarator_span, "previous declarator ends here");
}
builder
.primary_label(blame_span, primary_text)
.code("P0053")
.build()
}
/// P0054
pub(super) fn diagnostic_variable_declarator_array_size_missing_closing_bracket<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let open_bracket_span = related_span(&error, OPEN_BRACKET);
let blame_span = error.blame_span;
let primary_text = expected_before_found(file, blame_span, "expected `]`");
let primary_span =
primary_span_with_optional_multiline_context(file, open_bracket_span, blame_span);
let mut builder = DiagnosticBuilder::error("missing `]` to close array size expression");
if should_label_keyword(
file,
declaration_keyword_span,
open_bracket_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(open_bracket_span) = open_bracket_span {
if should_show_context_label(file, open_bracket_span, blame_span) {
builder =
builder.secondary_label(open_bracket_span, "array size expression starts here");
}
}
builder
.primary_label(primary_span, primary_text)
.code("P0054")
.build()
}
/// P0055
pub(super) fn diagnostic_variable_declarator_array_size_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let variable_name_span = related_span(
&error,
diagnostic_labels::EXPRESSION_REQUIRED_BY,
);
let expected_after_span = related_span(
&error,
diagnostic_labels::EXPRESSION_EXPECTED_AFTER,
);
let blame_span = error.blame_span;
let variable_name = variable_name_span.and_then(|span| match found_at(file, span.start) {
FoundAt::Token(token_text) => Some(token_text),
FoundAt::EndOfFile | FoundAt::Unknown => None,
});
let title = match variable_name {
Some(variable_name) => {
format!("expected array size expression for `{variable_name}` between `[` and `]`")
}
None => "expected array size expression between `[` and `]`".to_string(),
};
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => {
format!("expected array size expression before `{token_text}`")
}
FoundAt::EndOfFile => "expected array size expression before end of file".to_string(),
FoundAt::Unknown => "expected array size expression here".to_string(),
};
let primary_span =
primary_span_with_optional_multiline_context(file, expected_after_span, blame_span);
let mut builder = DiagnosticBuilder::error(title);
if let Some(expected_after_span) = expected_after_span {
if should_show_context_label(file, expected_after_span, blame_span) {
builder = builder.secondary_label(
expected_after_span,
"after this `[`, an array size expression was expected",
);
}
}
if let Some(variable_name_span) = variable_name_span {
let variable_name_is_already_visible =
file.same_line(variable_name_span.start, blame_span.start)
|| file.same_line(variable_name_span.start, blame_span.end)
|| expected_after_span.is_some_and(|expected_after_span| {
file.same_line(variable_name_span.start, expected_after_span.start)
|| file.same_line(variable_name_span.start, expected_after_span.end)
});
if should_show_context_label(file, variable_name_span, blame_span)
&& !variable_name_is_already_visible
{
builder = builder.secondary_label(
variable_name_span,
"this variable has an array size suffix",
);
}
}
builder
.primary_label(primary_span, primary_text)
.code("P0055")
.build()
}
/// P0056
pub(super) fn diagnostic_variable_declarator_initializer_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let variable_name_span = related_span(
&error,
diagnostic_labels::EXPRESSION_REQUIRED_BY,
);
let expected_after_span = related_span(
&error,
diagnostic_labels::EXPRESSION_EXPECTED_AFTER,
);
let blame_span = error.blame_span;
let variable_name = variable_name_span.and_then(|span| match found_at(file, span.start) {
FoundAt::Token(token_text) => Some(token_text),
FoundAt::EndOfFile | FoundAt::Unknown => None,
});
let title = match (variable_name, found_at(file, blame_span.start)) {
(Some(variable_name), FoundAt::Token(token_text)) => {
format!(
"expected initializer expression for `{variable_name}` after `=`, found `{token_text}`"
)
}
(Some(variable_name), FoundAt::EndOfFile) => {
format!(
"expected initializer expression for `{variable_name}` after `=`, found end of file"
)
}
(Some(variable_name), FoundAt::Unknown) => {
format!("expected initializer expression for `{variable_name}` after `=`")
}
(None, FoundAt::Token(token_text)) => {
format!("expected initializer expression after `=`, found `{token_text}`")
}
(None, FoundAt::EndOfFile) => {
"expected initializer expression after `=`, found end of file".to_string()
}
(None, FoundAt::Unknown) => "expected initializer expression after `=`".to_string(),
};
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => format!("unexpected `{token_text}`"),
FoundAt::EndOfFile => "reached end of file here".to_string(),
FoundAt::Unknown => "expected initializer expression here".to_string(),
};
let mut builder = DiagnosticBuilder::error(title);
if let Some(expected_after_span) = expected_after_span {
if should_show_context_label(file, expected_after_span, blame_span) {
builder = builder.secondary_label(
expected_after_span,
"after this `=`, an initializer expression was expected",
);
}
}
if let Some(variable_name_span) = variable_name_span {
let variable_name_is_already_visible =
file.same_line(variable_name_span.start, blame_span.start)
|| file.same_line(variable_name_span.start, blame_span.end)
|| expected_after_span.is_some_and(|expected_after_span| {
file.same_line(variable_name_span.start, expected_after_span.start)
|| file.same_line(variable_name_span.start, expected_after_span.end)
});
if should_show_context_label(file, variable_name_span, blame_span)
&& !variable_name_is_already_visible
{
builder = builder.secondary_label(
variable_name_span,
"initializer is for this variable",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0056")
.build()
}

24
rottlib/src/diagnostics/parse_error_diagnostics/mod.rs
View File

@ -19,7 +19,6 @@ use crate::parser::{ParseError, ParseErrorKind};
mod block_items; mod block_items;
mod control_flow_expressions; mod control_flow_expressions;
mod declarations;
mod primary_expressions; mod primary_expressions;
mod selector_expressions; mod selector_expressions;
mod switch_expressions; mod switch_expressions;
@ -78,7 +77,6 @@ pub(crate) fn diagnostic_from_parse_error<'src>(
file: &TokenizedFile<'src>, file: &TokenizedFile<'src>,
) -> Diagnostic { ) -> Diagnostic {
use control_flow_expressions::*; use control_flow_expressions::*;
use declarations::*;
use primary_expressions::*; use primary_expressions::*;
use selector_expressions::*; use selector_expressions::*;
use switch_expressions::*; use switch_expressions::*;
@ -191,28 +189,6 @@ pub(crate) fn diagnostic_from_parse_error<'src>(
ParseErrorKind::SwitchCaseExpressionInvalidStart => { ParseErrorKind::SwitchCaseExpressionInvalidStart => {
diagnostic_switch_case_expression_invalid_start(error, file) diagnostic_switch_case_expression_invalid_start(error, file)
} }
// declarations.rs
ParseErrorKind::LocalVariableDeclarationMissingSemicolon => {
diagnostic_local_variable_declaration_missing_semicolon(error, file)
}
ParseErrorKind::VariableDeclaratorListEmpty => {
diagnostic_variable_declarator_list_empty(error, file)
}
ParseErrorKind::VariableDeclaratorExpected => {
diagnostic_variable_declarator_expected(error, file)
}
ParseErrorKind::VariableDeclaratorListMissingSeparator => {
diagnostic_variable_declarator_list_missing_separator(error, file)
}
ParseErrorKind::VariableDeclaratorArraySizeMissingClosingBracket => {
diagnostic_variable_declarator_array_size_missing_closing_bracket(error, file)
}
ParseErrorKind::VariableDeclaratorArraySizeExpected => {
diagnostic_variable_declarator_array_size_expected(error, file)
}
ParseErrorKind::VariableDeclaratorInitializerExpected => {
diagnostic_variable_declarator_initializer_expected(error, file)
}
_ => DiagnosticBuilder::error(format!("error {:?} while parsing", error.kind)) _ => DiagnosticBuilder::error(format!("error {:?} while parsing", error.kind))
.primary_label(error.covered_span, "happened here") .primary_label(error.covered_span, "happened here")

7
rottlib/src/diagnostics/parse_error_diagnostics/primary_expressions.rs
View File

@ -1,6 +1,6 @@
use super::{Diagnostic, DiagnosticBuilder, FoundAt, collapse_span_to_end_on_same_line, found_at}; use super::{Diagnostic, DiagnosticBuilder, FoundAt, collapse_span_to_end_on_same_line, found_at};
use crate::lexer::{TokenSpan, TokenizedFile}; use crate::lexer::{TokenSpan, TokenizedFile};
use crate::parser::{ParseError, diagnostic_labels}; use crate::parser::ParseError;
pub(super) fn diagnostic_parenthesized_expression_invalid_start<'src>( pub(super) fn diagnostic_parenthesized_expression_invalid_start<'src>(
error: ParseError, error: ParseError,
@ -164,10 +164,7 @@ pub(super) fn diagnostic_class_type_expected_qualified_type_name<'src>(
error: ParseError, error: ParseError,
file: &TokenizedFile<'src>, file: &TokenizedFile<'src>,
) -> Diagnostic { ) -> Diagnostic {
let qualifier_dot_span = error let qualifier_dot_span = error.related_spans.get("qualifier_dot").copied();
.related_spans
.get(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT)
.copied();
let class_span = error.related_spans.get("class_keyword").copied(); let class_span = error.related_spans.get("class_keyword").copied();
let blame_pos = error.blame_span.end; let blame_pos = error.blame_span.end;

6
rottlib/src/lexer/token.rs
View File

@ -1,10 +1,10 @@
//! Token definitions for Fermented UnrealScript. //! Token definitions for Fermented `UnrealScript`.
//! //!
//! These are the tokens consumed by the parser and derived from [`RawToken`]s. //! These are the tokens consumed by the parser and derived from [`RawToken`]s.
use super::{BraceKind, raw_lexer::RawToken}; use super::{BraceKind, raw_lexer::RawToken};
/// Tokens consumed by the Fermented UnrealScript parser. /// Tokens consumed by the Fermented `UnrealScript` parser.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Token { pub enum Token {
ExecDirective, ExecDirective,
@ -408,7 +408,7 @@ impl Token {
} }
} }
/// Reserved words of Fermented UnrealScript. /// Reserved words of Fermented `UnrealScript`.
/// ///
/// These are represented in [`Token`] as [`Token::Keyword`]. /// These are represented in [`Token`] as [`Token::Keyword`].
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]

3
rottlib/src/parser/cursor.rs
View File

@ -321,7 +321,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.advance(); self.advance();
Ok(token_position) Ok(token_position)
} else { } else {
let anchor = self.peek_position_or_eof(); let anchor = self.peek_position().unwrap_or_else(|| self.file.eof());
Err(self Err(self
.make_error_at(error_kind, anchor) .make_error_at(error_kind, anchor)
.blame(TokenSpan::new(anchor))) .blame(TokenSpan::new(anchor)))
@ -378,7 +378,6 @@ impl<'src, 'arena> Parser<'src, 'arena> {
"parser made no forward progress" "parser made no forward progress"
); );
if peeked_position <= old_position { if peeked_position <= old_position {
// TODO: check if we're in error and panic if not
self.advance(); self.advance();
} }
if self.file.is_eof(&old_position) { if self.file.is_eof(&old_position) {

66
rottlib/src/parser/errors.rs
View File

@ -1,6 +1,6 @@
//! Submodule with parsing related errors. //! Submodule with parsing related errors.
use std::collections::{HashMap, HashSet}; use std::collections::HashMap;
use crate::{lexer::TokenPosition, lexer::TokenSpan}; use crate::{lexer::TokenPosition, lexer::TokenSpan};
@ -100,63 +100,27 @@ pub enum ParseErrorKind {
SwitchCaseMissingExpression, SwitchCaseMissingExpression,
/// P0042 /// P0042
SwitchCaseExpressionInvalidStart, SwitchCaseExpressionInvalidStart,
/// P0043
InvalidNumericLiteral,
/// P0044
EnumMissingName,
/// P0045
EnumMissingOpeningBrace,
/// P0046
EnumMissingClosingBrace,
/// P0047
EnumExpectedCommaOrClosingBraceAfterVariant,
/// P0048
EnumVariantInvalidStart,
/// P0049
EnumExpectedVariant,
/// P0050
LocalVariableDeclarationMissingSemicolon,
/// P0051
VariableDeclaratorListEmpty,
/// P0052
VariableDeclaratorExpected,
/// P0053
VariableDeclaratorListMissingSeparator,
/// P0054
VariableDeclaratorArraySizeMissingClosingBracket,
/// P0055
VariableDeclaratorArraySizeExpected,
/// P0056
VariableDeclaratorInitializerExpected,
/// P0057
VarEditorSpecifierExpected,
/// P0058
VarEditorSpecifierListMissingSeparator,
/// P0059
VarEditorSpecifierListExpectedCommaOrClosingParenthesis,
/// P0060
VarEditorSpecifierListMissingClosingParenthesis,
/// P0061
TypeSpecifierExpected,
/// P0062
NamedTypeInvalidName,
/// P0063
ArrayTypeMissingOpeningAngleBracket,
/// P0064
ArrayTypeMissingClosingAngleBracket,
/// P0065
ArrayTypeMissingElementType,
// ================== Old errors to be thrown away! ================== // ================== Old errors to be thrown away! ==================
/// Found an unexpected token while parsing an expression. /// Found an unexpected token while parsing an expression.
ExpressionUnexpectedToken, ExpressionUnexpectedToken,
DeclEmptyVariableDeclarations,
DeclNoSeparatorBetweenVariableDeclarations,
DeclExpectedRightBracketAfterArraySize,
DeclExpectedCommaAfterVariableDeclarator, DeclExpectedCommaAfterVariableDeclarator,
TypeSpecExpectedType,
TypeSpecInvalidNamedTypeName,
TypeSpecArrayMissingOpeningAngle,
TypeSpecArrayMissingInnerType, TypeSpecArrayMissingInnerType,
TypeSpecArrayMissingClosingAngle,
TypeSpecClassMissingInnerType, TypeSpecClassMissingInnerType,
TypeSpecClassMissingClosingAngle, TypeSpecClassMissingClosingAngle,
BlockMissingSemicolonAfterStatement, BlockMissingSemicolonAfterStatement,
/// Unexpected end of input while parsing. /// Unexpected end of input while parsing.
UnexpectedEndOfFile, UnexpectedEndOfFile,
/// Token looked like a numeric literal but could not be parsed as one.
InvalidNumericLiteral,
/// A bare expression appeared in a `switch` arm but was not the final arm. /// A bare expression appeared in a `switch` arm but was not the final arm.
/// ///
/// Such an expression must be terminated with `;` or be the final arm. /// Such an expression must be terminated with `;` or be the final arm.
@ -176,6 +140,8 @@ pub enum ParseErrorKind {
/// ///
/// At least one variable name must follow the type. /// At least one variable name must follow the type.
DeclMissingIdentifier, DeclMissingIdentifier,
/// Invalid variable name identifier in non-`local` variable definition.
DeclBadVariableIdentifier,
/// Found an unexpected token while parsing a declaration literal. /// Found an unexpected token while parsing a declaration literal.
/// ///
/// Expected one of: integer, float, string, `true`, `false`, `none` /// Expected one of: integer, float, string, `true`, `false`, `none`
@ -240,12 +206,15 @@ pub enum ParseErrorKind {
FunctionParamsMissingOpeningParenthesis, FunctionParamsMissingOpeningParenthesis,
FunctionParamsMissingClosingParenthesis, FunctionParamsMissingClosingParenthesis,
ClassUnexpectedItem, ClassUnexpectedItem,
EnumMissingLeftBrace,
EnumBadVariant, EnumBadVariant,
StructFieldMissingName, StructFieldMissingName,
StructFieldMissingSemicolon, StructFieldMissingSemicolon,
StructMissingRightBrace, StructMissingRightBrace,
// Named enum/struct typedefs // Named enum/struct typedefs
EnumMissingKeyword, // class member: expected `enum` EnumMissingKeyword, // class member: expected `enum`
EnumExpectedNameOrBrace, // after `enum`, expected identifier
EnumNoClosingBrace,
EnumEmptyVariants, EnumEmptyVariants,
EnumNoSeparatorBetweenVariants, EnumNoSeparatorBetweenVariants,
EnumMissingLBrace, EnumMissingLBrace,
@ -306,7 +275,6 @@ pub enum ParseErrorKind {
StateParensMissingRParen, StateParensMissingRParen,
BadTypeInClassTypeDeclaration, BadTypeInClassTypeDeclaration,
IdentifierExpected, IdentifierExpected,
VariableDeclaratorBadIdentifier,
// --- Generic list diagnostics (comma-separated, closed by `)`) --- // --- Generic list diagnostics (comma-separated, closed by `)`) ---
/// Saw `)` immediately after `(`, or closed the list without any items. /// Saw `)` immediately after `(`, or closed the list without any items.
@ -370,7 +338,6 @@ pub struct ParseError {
/// The source span in which the error was detected. /// The source span in which the error was detected.
pub covered_span: TokenSpan, pub covered_span: TokenSpan,
pub related_spans: HashMap<String, TokenSpan>, pub related_spans: HashMap<String, TokenSpan>,
pub flags: HashSet<String>,
} }
pub type ParseResult<'src, 'arena, T> = Result<T, ParseError>; pub type ParseResult<'src, 'arena, T> = Result<T, ParseError>;
@ -391,7 +358,6 @@ impl crate::parser::Parser<'_, '_> {
blame_span: TokenSpan::new(position), blame_span: TokenSpan::new(position),
covered_span: TokenSpan::new(position), covered_span: TokenSpan::new(position),
related_spans: HashMap::new(), related_spans: HashMap::new(),
flags: HashSet::new(),
} }
} }
} }

17
rottlib/src/parser/grammar/class.rs
View File

@ -12,7 +12,7 @@ use crate::ast::{
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan}; use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, ResultRecoveryExt, SyncLevel}; use crate::parser::{ParseErrorKind, ParseResult, ResultRecoveryExt, SyncLevel};
use std::collections::{HashMap, HashSet}; use std::collections::HashMap;
impl<'src, 'arena> crate::parser::Parser<'src, 'arena> { impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
#[inline] #[inline]
@ -744,7 +744,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
let is_negative = matches!(token, Token::Minus); let is_negative = matches!(token, Token::Minus);
self.advance(); self.advance();
let (next_token, next_lexeme, token_position) = let (next_token, next_lexeme, _) =
self.require_token_lexeme_and_position(ParseErrorKind::InvalidNumericLiteral)?; self.require_token_lexeme_and_position(ParseErrorKind::InvalidNumericLiteral)?;
match next_token { match next_token {
@ -761,7 +761,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
let mut signed_lexeme = String::with_capacity(1 + next_lexeme.len()); let mut signed_lexeme = String::with_capacity(1 + next_lexeme.len());
signed_lexeme.push(if is_negative { '-' } else { '+' }); signed_lexeme.push(if is_negative { '-' } else { '+' });
signed_lexeme.push_str(next_lexeme); signed_lexeme.push_str(next_lexeme);
let value = self.decode_float_literal(&signed_lexeme, token_position)?; let value = self.decode_float_literal(&signed_lexeme)?;
self.advance(); self.advance();
DeclarationLiteral::Float(value) DeclarationLiteral::Float(value)
} }
@ -778,7 +778,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
DeclarationLiteral::Integer(value) DeclarationLiteral::Integer(value)
} }
Token::FloatLiteral => { Token::FloatLiteral => {
let value = self.decode_float_literal(lexeme, token_position)?; let value = self.decode_float_literal(lexeme)?;
self.advance(); self.advance();
DeclarationLiteral::Float(value) DeclarationLiteral::Float(value)
} }
@ -844,8 +844,8 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
ParseErrorKind::ClassUnexpectedItem, ParseErrorKind::ClassUnexpectedItem,
)?; )?;
let name = self.parse_identifier(ParseErrorKind::VariableDeclaratorBadIdentifier)?; let name = self.parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)?;
self.expect(Token::Assign, ParseErrorKind::NamedTypeInvalidName)?; self.expect(Token::Assign, ParseErrorKind::TypeSpecInvalidNamedTypeName)?;
let value = self.parse_declaration_literal_class()?; let value = self.parse_declaration_literal_class()?;
self.expect(Token::Semicolon, ParseErrorKind::DeclMissingSemicolon)?; self.expect(Token::Semicolon, ParseErrorKind::DeclMissingSemicolon)?;
@ -865,7 +865,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
match self.peek_token_and_position() { match self.peek_token_and_position() {
Some((next_token, declarator_start)) if next_token.is_valid_identifier_name() => { Some((next_token, declarator_start)) if next_token.is_valid_identifier_name() => {
let identifier = self let identifier = self
.parse_identifier(ParseErrorKind::VariableDeclaratorBadIdentifier) .parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)
.unwrap_or(IdentifierToken(declarator_start)); .unwrap_or(IdentifierToken(declarator_start));
let array_size = match self.parse_array_len_expr() { let array_size = match self.parse_array_len_expr() {
@ -898,7 +898,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
break; break;
} }
Some((_, _)) if declarators.is_empty() => { Some((_, _)) if declarators.is_empty() => {
self.report_error_here(ParseErrorKind::VariableDeclaratorBadIdentifier); self.report_error_here(ParseErrorKind::DeclBadVariableIdentifier);
self.recover_until(SyncLevel::StatementStart); self.recover_until(SyncLevel::StatementStart);
let _ = self.eat(Token::Semicolon); let _ = self.eat(Token::Semicolon);
break; break;
@ -951,7 +951,6 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
blame_span: TokenSpan::range(list_start, list_end), blame_span: TokenSpan::range(list_start, list_end),
covered_span: TokenSpan::range(list_start, list_end), covered_span: TokenSpan::range(list_start, list_end),
related_spans: HashMap::new(), related_spans: HashMap::new(),
flags: HashSet::new(),
}); });
} }

35
rottlib/src/parser/grammar/declarations/enum_definition.rs
View File

@ -1,11 +1,11 @@
//! Parsing of enum definitions for Fermented UnrealScript. //! Parsing of enum definitions for Fermented `UnrealScript`.
use std::ops::ControlFlow; use std::ops::ControlFlow;
use crate::arena::ArenaVec; use crate::arena::ArenaVec;
use crate::ast::{EnumDefRef, EnumDefinition, IdentifierToken}; use crate::ast::{EnumDefRef, EnumDefinition, IdentifierToken};
use crate::lexer::Token; use crate::lexer::Token;
use crate::lexer::{TokenPosition, TokenSpan}; use crate::lexer::{TokenSpan, TokenPosition};
use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel}; use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
@ -24,19 +24,12 @@ impl<'src, 'arena> Parser<'src, 'arena> {
enum_keyword_position: TokenPosition, enum_keyword_position: TokenPosition,
) -> EnumDefRef<'src, 'arena> { ) -> EnumDefRef<'src, 'arena> {
let name = self let name = self
.parse_identifier(ParseErrorKind::EnumMissingName) .parse_identifier(ParseErrorKind::EnumExpectedNameOrBrace)
.unwrap_or_fallback(self); .unwrap_or_fallback(self);
self.expect(Token::LeftBrace, ParseErrorKind::EnumMissingOpeningBrace) self.expect(Token::LeftBrace, ParseErrorKind::EnumMissingLeftBrace)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self); .report_error(self);
let variants = self.parse_enum_variants(); let variants = self.parse_enum_variants();
if variants.is_empty() { self.expect(Token::RightBrace, ParseErrorKind::EnumNoClosingBrace)
self.make_error_at_last_consumed(ParseErrorKind::EnumExpectedVariant)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self);
}
self.expect(Token::RightBrace, ParseErrorKind::EnumMissingClosingBrace)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self); .report_error(self);
let span = TokenSpan::range( let span = TokenSpan::range(
@ -95,10 +88,6 @@ impl<'src, 'arena> Parser<'src, 'arena> {
} }
self.make_error_at_last_consumed(ParseErrorKind::EnumEmptyVariants) self.make_error_at_last_consumed(ParseErrorKind::EnumEmptyVariants)
.widen_error_span_from(error_start_position) .widen_error_span_from(error_start_position)
.blame(TokenSpan::range(
error_start_position,
self.last_consumed_position_or_start(),
))
.report_error(self); .report_error(self);
if matches!(self.peek_token(), Some(Token::RightBrace) | None) { if matches!(self.peek_token(), Some(Token::RightBrace) | None) {
ControlFlow::Break(()) ControlFlow::Break(())
@ -114,7 +103,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
variants: &mut ArenaVec<'arena, IdentifierToken>, variants: &mut ArenaVec<'arena, IdentifierToken>,
) -> ControlFlow<()> { ) -> ControlFlow<()> {
self.parse_identifier(ParseErrorKind::EnumVariantInvalidStart) self.parse_identifier(ParseErrorKind::EnumBadVariant)
.sync_error_until(self, SyncLevel::StatementStart) .sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self) .ok_or_report(self)
.map_or(ControlFlow::Break(()), |variant| { .map_or(ControlFlow::Break(()), |variant| {
@ -131,7 +120,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
variants: &mut ArenaVec<'arena, IdentifierToken>, variants: &mut ArenaVec<'arena, IdentifierToken>,
) -> ControlFlow<()> { ) -> ControlFlow<()> {
let Some(variant) = self let Some(variant) = self
.parse_identifier(ParseErrorKind::EnumExpectedCommaOrClosingBraceAfterVariant) .parse_identifier(ParseErrorKind::EnumBadVariant)
.widen_error_span_from(error_start_position) .widen_error_span_from(error_start_position)
.sync_error_until(self, SyncLevel::StatementStart) .sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self) .ok_or_report(self)
@ -139,13 +128,9 @@ impl<'src, 'arena> Parser<'src, 'arena> {
// If we don't even get a good identifier - error is different // If we don't even get a good identifier - error is different
return ControlFlow::Break(()); return ControlFlow::Break(());
}; };
self.make_error_at_last_consumed(ParseErrorKind::EnumNoSeparatorBetweenVariants)
let mut error = self.make_error_at_last_consumed(ParseErrorKind::EnumNoSeparatorBetweenVariants) .widen_error_span_from(error_start_position)
.widen_error_span_from(error_start_position); .report_error(self);
if let Some(previous_variant) = variants.last() {
error = error.related("previous_variant", previous_variant.span());
}
error.report_error(self);
variants.push(variant); variants.push(variant);
ControlFlow::Continue(()) ControlFlow::Continue(())

2
rottlib/src/parser/grammar/declarations/mod.rs
View File

@ -1,4 +1,4 @@
//! Declaration parsing for Fermented UnrealScript. //! Declaration parsing for Fermented `UnrealScript`.
//! //!
//! Implements recursive-descent parsing for declaration-related grammar: //! Implements recursive-descent parsing for declaration-related grammar:
//! type specifiers, enum and struct definitions, `var(...)` prefixes, //! type specifiers, enum and struct definitions, `var(...)` prefixes,

8
rottlib/src/parser/grammar/declarations/struct_definition.rs
View File

@ -1,8 +1,8 @@
//! Parsing of struct definitions for Fermented UnrealScript. //! Parsing of struct definitions for Fermented `UnrealScript`.
//! //!
//! ## C++ block handling //! ## C++ block handling
//! //!
//! The Fermented UnrealScript parser must support parsing several legacy //! The Fermented `UnrealScript` parser must support parsing several legacy
//! source files that contain `cpptext` or `cppstruct`. Our compiler does not //! source files that contain `cpptext` or `cppstruct`. Our compiler does not
//! compile with C++ code and therefore does not need these blocks in //! compile with C++ code and therefore does not need these blocks in
//! the resulting AST. We treat them the same as trivia and skip them. //! the resulting AST. We treat them the same as trivia and skip them.
@ -120,9 +120,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
let Some(field_prefix) = self.parse_struct_field_prefix() else { let Some(field_prefix) = self.parse_struct_field_prefix() else {
return StructBodyItemParseOutcome::Skip; return StructBodyItemParseOutcome::Skip;
}; };
// TODO: correct span let declarators = self.parse_variable_declarators();
let declarators = self
.parse_variable_declarators(var_keyword_position, TokenSpan::new(var_keyword_position));
if !self.eat(Token::Semicolon) { if !self.eat(Token::Semicolon) {
self.report_error_here(ParseErrorKind::StructFieldMissingSemicolon); self.report_error_here(ParseErrorKind::StructFieldMissingSemicolon);
self.recover_until(SyncLevel::BlockBoundary); self.recover_until(SyncLevel::BlockBoundary);

64
rottlib/src/parser/grammar/declarations/type_specifier.rs
View File

@ -1,8 +1,8 @@
//! Parsing of type specifiers for Fermented UnrealScript. //! Parsing of type specifiers for Fermented `UnrealScript`.
use crate::ast::{TypeSpecifier, TypeSpecifierRef}; use crate::ast::{TypeSpecifier, TypeSpecifierRef};
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan}; use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, SyncLevel}; use crate::parser::{ParseErrorKind, ParseResult, Parser};
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a type specifier used in variable declarations. /// Parses a type specifier used in variable declarations.
@ -15,7 +15,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> { ) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
let (starting_token, starting_token_position) = let (starting_token, starting_token_position) =
self.require_token_and_position(ParseErrorKind::TypeSpecifierExpected)?; self.require_token_and_position(ParseErrorKind::TypeSpecExpectedType)?;
match starting_token { match starting_token {
Token::Keyword(Keyword::Enum) => { Token::Keyword(Keyword::Enum) => {
@ -36,16 +36,13 @@ impl<'src, 'arena> Parser<'src, 'arena> {
} }
_ if starting_token.is_valid_type_name() => { _ if starting_token.is_valid_type_name() => {
let type_name = let type_name =
self.parse_qualified_identifier(ParseErrorKind::NamedTypeInvalidName)?; self.parse_qualified_identifier(ParseErrorKind::TypeSpecInvalidNamedTypeName)?;
let full_span = *type_name.span(); let full_span = *type_name.span();
Ok(self Ok(self
.arena .arena
.alloc_node(TypeSpecifier::Named(type_name), full_span)) .alloc_node(TypeSpecifier::Named(type_name), full_span))
} }
_ => Err(self.make_error_at( _ => Err(self.make_error_at_last_consumed(ParseErrorKind::TypeSpecExpectedType)),
ParseErrorKind::TypeSpecifierExpected,
starting_token_position,
)),
} }
} }
@ -73,41 +70,18 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
starting_token_position: TokenPosition, starting_token_position: TokenPosition,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> { ) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
let left_angle_bracket = self.expect( self.expect(
Token::Less, Token::Less,
ParseErrorKind::ArrayTypeMissingOpeningAngleBracket, ParseErrorKind::TypeSpecArrayMissingOpeningAngle,
)?; )?;
if self.eat(Token::Greater) {
return Err(
self.make_error_at_last_consumed(ParseErrorKind::ArrayTypeMissingElementType)
);
}
let element_modifiers = self.parse_var_declaration_modifiers(); let element_modifiers = self.parse_var_declaration_modifiers();
// Don't sync on this error, because for a better recovery in likely let element_type = self.parse_type_specifier()?;
// case of missing `>` we need to check whether inserting it would fix let closing_angle_bracket_position = self.expect(
// the issue.
let element_type = self.parse_type_specifier().unwrap_or_fallback(self);
// A type-closing `>` can be followed up by either another `>` or
// identifier, so on failure check whether we have an identifier next.
let closing_angle_bracket_position = match self.expect(
Token::Greater, Token::Greater,
ParseErrorKind::ArrayTypeMissingClosingAngleBracket, ParseErrorKind::TypeSpecArrayMissingClosingAngle,
) { )?;
Ok(position) => position,
Err(error) => {
let next_token_can_start_declarator = self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name());
let error = if next_token_can_start_declarator {
error
} else {
error.sync_error_at(self, SyncLevel::CloseAngleBracket)
};
Err(error).unwrap_or_fallback(self)
}
};
let array_span = TokenSpan::range(starting_token_position, closing_angle_bracket_position); let array_span = TokenSpan::range(starting_token_position, closing_angle_bracket_position);
Ok(self.arena.alloc_node( Ok(self.arena.alloc_node(
TypeSpecifier::Array { TypeSpecifier::Array {
element_type, element_type,
@ -121,13 +95,15 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
starting_token_position: TokenPosition, starting_token_position: TokenPosition,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> { ) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
let (inner_type_name, class_type_end) = let (inner_type_name, class_type_end) = if self.eat(Token::Less) {
if let Some(left_angle_bracket_position) = self.eat_with_position(Token::Less) { let inner_type_name = Some(
let parsed = self.parse_class_type_argument_tail( self.parse_qualified_identifier(ParseErrorKind::TypeSpecClassMissingInnerType)?,
starting_token_position, );
left_angle_bracket_position, let class_type_end = self.expect(
Token::Greater,
ParseErrorKind::TypeSpecClassMissingClosingAngle,
)?; )?;
(Some(parsed.type_name), parsed.right_angle_bracket_position) (inner_type_name, class_type_end)
} else { } else {
(None, starting_token_position) (None, starting_token_position)
}; };

252
rottlib/src/parser/grammar/declarations/var_specifiers.rs
View File

@ -1,56 +1,11 @@
//! Parsing of declaration specifiers used in `var(...) ...` syntax for //! Parsing of declaration specifiers used in `var(...) ...` syntax for
//! Fermented UnrealScript. //! Fermented `UnrealScript`.
use std::ops::ControlFlow;
use crate::arena::ArenaVec; use crate::arena::ArenaVec;
use crate::ast::{VarEditorSpecifier, VarEditorSpecifierRef, VarModifier}; use crate::ast::{VarEditorSpecifier, VarEditorSpecifierRef, VarModifier};
use crate::lexer::{Token, TokenPosition, TokenSpan}; use crate::lexer::Token;
use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel}; use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum VarEditorSpecifierListParseStep {
ExpectingSpecifier,
ExpectingSeparator,
}
struct VarEditorSpecifierListParseState<'arena, SpecifierRef> {
editor_specifiers: ArenaVec<'arena, SpecifierRef>,
open_parenthesis_position: TokenPosition,
pending_separator_position: Option<TokenPosition>,
previous_specifier_span: Option<TokenSpan>,
parse_step: VarEditorSpecifierListParseStep,
}
impl<'arena, SpecifierRef> VarEditorSpecifierListParseState<'arena, SpecifierRef> {
#[must_use]
fn new<'src>(parser: &Parser<'src, 'arena>, open_parenthesis_position: TokenPosition) -> Self {
Self {
editor_specifiers: parser.arena.vec(),
open_parenthesis_position,
pending_separator_position: None,
previous_specifier_span: None,
parse_step: VarEditorSpecifierListParseStep::ExpectingSpecifier,
}
}
fn set_specifier_expected(&mut self, separator_position: TokenPosition) {
self.pending_separator_position = Some(separator_position);
self.parse_step = VarEditorSpecifierListParseStep::ExpectingSpecifier;
}
fn set_separator_expected(&mut self) {
self.pending_separator_position = None;
self.parse_step = VarEditorSpecifierListParseStep::ExpectingSeparator;
}
fn push_specifier(&mut self, specifier: SpecifierRef, specifier_span: TokenSpan) {
self.previous_specifier_span = Some(specifier_span);
self.editor_specifiers.push(specifier);
self.set_separator_expected();
}
}
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a consecutive run of variable declaration modifiers. /// Parses a consecutive run of variable declaration modifiers.
/// ///
@ -65,201 +20,70 @@ impl<'src, 'arena> Parser<'src, 'arena> {
#[must_use] #[must_use]
pub(crate) fn parse_var_declaration_modifiers(&mut self) -> ArenaVec<'arena, VarModifier> { pub(crate) fn parse_var_declaration_modifiers(&mut self) -> ArenaVec<'arena, VarModifier> {
let mut modifiers = self.arena.vec(); let mut modifiers = self.arena.vec();
while let Some(current_token_and_position) = self.peek_token_and_position() while let Some(current_token_and_position) = self.peek_token_and_position() {
&& let Ok(parsed_modifier) = VarModifier::try_from(current_token_and_position) let Ok(parsed_modifier) = VarModifier::try_from(current_token_and_position) else {
{ break;
};
self.advance(); self.advance();
modifiers.push(parsed_modifier); modifiers.push(parsed_modifier);
} }
modifiers modifiers
} }
// TODO: should this have "optional" in the name?
/// Parses the optional parenthesized editor specifier list in `var(...)`. /// Parses the optional parenthesized editor specifier list in `var(...)`.
/// ///
/// Assumes that `var` has already been consumed. /// Assumes that `var` has already been consumed.
/// ///
/// Returns `None` if the current token is not `(`. Returns `Some(...)` once /// Returns `None` if the current token is not `(`. Returns `Some(...)` once
/// `(` is present, including for an empty list. /// `(` is present, including for an empty list.
///
/// Recovery is intentionally minimal because these specifier lists are not
/// important enough to justify aggressive repair.
#[must_use] #[must_use]
pub(crate) fn parse_var_editor_specifier_list( pub(crate) fn parse_var_editor_specifier_list(
&mut self, &mut self,
) -> Option<ArenaVec<'arena, VarEditorSpecifierRef<'src, 'arena>>> { ) -> Option<ArenaVec<'arena, VarEditorSpecifierRef<'src, 'arena>>> {
use VarEditorSpecifierListParseStep::{ExpectingSeparator, ExpectingSpecifier}; if !self.eat(Token::LeftParenthesis) {
let Some(left_parenthesis_position) = self.eat_with_position(Token::LeftParenthesis) else {
return None; return None;
};
let mut state = VarEditorSpecifierListParseState::new(self, left_parenthesis_position);
// Progress invariant: every branch that continues the loop consumes at
// least one token. Branches that only report an error must stop the
// loop or synchronize before continuing.
while let Some((next_token, next_token_position)) = self.peek_token_and_position() {
match (state.parse_step, next_token) {
(ExpectingSpecifier, Token::RightParenthesis) => {
if state.pending_separator_position.is_some() {
self.report_missing_var_editor_specifier_at_current_token(&state);
} }
break; let mut editor_specifiers = self.arena.vec();
} while let Some((next_token, next_token_lexeme, next_token_position)) =
(ExpectingSpecifier, Token::Comma) => { self.peek_token_lexeme_and_position()
if self && next_token != Token::RightParenthesis
.recover_empty_var_editor_specifier_run(next_token_position, &mut state)
.is_break()
{ {
break; if next_token == Token::StringLiteral {
} self.advance();
} let string_value = self.unescape_string_literal(next_token_lexeme);
(ExpectingSpecifier, _) => { editor_specifiers.push(self.arena.alloc_node_at(
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() { VarEditorSpecifier::String(string_value),
let parsed_specifier_span = *parsed_specifier.span(); next_token_position,
state.push_specifier(parsed_specifier, parsed_specifier_span); ));
} else if let Some(specifier_identifier) =
Self::identifier_token_from_token(next_token, next_token_position)
{
self.advance();
editor_specifiers.push(self.arena.alloc_node_at(
VarEditorSpecifier::Identifier(specifier_identifier),
next_token_position,
));
} else { } else {
self.report_missing_var_editor_specifier_at_current_token(&state); self.make_error_at_last_consumed(ParseErrorKind::VarSpecNotIdentifier)
break; .sync_error_until(self, SyncLevel::ListSeparator)
}
}
(ExpectingSeparator, Token::Comma) => {
self.advance(); // ','
state.set_specifier_expected(next_token_position);
}
(ExpectingSeparator, Token::RightParenthesis) => break,
(ExpectingSeparator, _) => {
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() {
let parsed_specifier_span = *parsed_specifier.span();
self.make_error_at_last_consumed(
ParseErrorKind::VarEditorSpecifierListMissingSeparator,
)
.widen_error_span_from(next_token_position)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.report_error(self); .report_error(self);
}
state.push_specifier(parsed_specifier, parsed_specifier_span); // Detailed recovery is not worthwhile here;
} else { // stop once list structure becomes unclear.
self.report_unexpected_token_after_var_editor_specifier(&state); if !self.eat(Token::Comma) {
break; break;
} }
}
}
// Guard against parser bugs that would otherwise leave declaration
// parsing stuck on the same token.
self.ensure_forward_progress(next_token_position); self.ensure_forward_progress(next_token_position);
} }
self.expect( self.expect(
Token::RightParenthesis, Token::RightParenthesis,
ParseErrorKind::VarEditorSpecifierListMissingClosingParenthesis, ParseErrorKind::VarSpecsMissingClosingParenthesis,
) )
.related_token("open_parenthesis", state.open_parenthesis_position) .sync_error_at(self, SyncLevel::CloseParenthesis)
.maybe_related("previous_specifier", state.previous_specifier_span)
.sync_error_at_matching_delimiter(self, left_parenthesis_position)
.report_error(self);
Some(state.editor_specifiers)
}
/// Tries to parse the current token as one `var(...)` editor specifier.
///
/// This is the only recognizer for editor specifier starts. It returns
/// [`None`] without consuming anything when the current token cannot start a
/// specifier.
fn try_parse_current_var_editor_specifier(
&mut self,
) -> Option<VarEditorSpecifierRef<'src, 'arena>> {
let (next_token, next_token_lexeme, next_token_position) =
self.peek_token_lexeme_and_position()?;
if next_token == Token::StringLiteral {
let string_value = self.unescape_string_literal(next_token_lexeme);
self.advance();
return Some(self.arena.alloc_node_at(
VarEditorSpecifier::String(string_value),
next_token_position,
));
}
let specifier_identifier =
Self::identifier_token_from_token(next_token, next_token_position)?;
self.advance();
Some(self.arena.alloc_node_at(
VarEditorSpecifier::Identifier(specifier_identifier),
next_token_position,
))
}
/// Reports a missing editor specifier at the current token.
fn report_missing_var_editor_specifier_at_current_token(
&mut self,
state: &VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) {
let position = self.peek_position_or_eof();
let missing_first_specifier =
state.editor_specifiers.is_empty() && state.pending_separator_position.is_none();
let mut error = self
.make_error_at(ParseErrorKind::VarEditorSpecifierExpected, position)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related_token("separator", state.pending_separator_position)
.maybe_related("previous_specifier", state.previous_specifier_span);
if missing_first_specifier {
error = error.with_flag("first_specifier");
}
error.report(self);
}
/// Recovers from one or more empty editor specifier slots produced by
/// commas.
///
/// Called when the current token is `,`. Returns [`ControlFlow::Continue`]
/// only when either the list can continue or a following specifier was
/// consumed as part of recovery.
fn recover_empty_var_editor_specifier_run(
&mut self,
error_start_position: TokenPosition,
state: &mut VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) -> ControlFlow<()> {
let missing_first_specifier =
state.editor_specifiers.is_empty() && state.pending_separator_position.is_none();
while let Some(Token::Comma) = self.peek_token() {
self.advance();
}
let mut error = self
.make_error_at_last_consumed(ParseErrorKind::VarEditorSpecifierExpected)
.widen_error_span_from(error_start_position)
.extend_blame_start_to_covered_start()
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related_token("separator", state.pending_separator_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.with_flag("empty_specifier_run");
if missing_first_specifier {
error = error.with_flag("first_specifier");
}
error.report_error(self);
if self.peek_token() == Some(Token::RightParenthesis) {
return ControlFlow::Break(());
}
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() {
let parsed_specifier_span = *parsed_specifier.span();
state.push_specifier(parsed_specifier, parsed_specifier_span);
ControlFlow::Continue(())
} else {
ControlFlow::Break(())
}
}
/// Reports a token that appears after an editor specifier where only `,`
/// or `)` can continue the list.
fn report_unexpected_token_after_var_editor_specifier(
&mut self,
state: &VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) {
let position = self.peek_position_or_eof();
self.make_error_at(
ParseErrorKind::VarEditorSpecifierListExpectedCommaOrClosingParenthesis,
position,
)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.sync_error_until(self, SyncLevel::CloseParenthesis)
.report_error(self); .report_error(self);
Some(editor_specifiers)
} }
} }

297
rottlib/src/parser/grammar/declarations/variable_declarators.rs
View File

@ -1,345 +1,172 @@
//! Parsing of comma-separated variable declarator lists for //! Parsing of comma-separated variable declarator lists for
//! Fermented UnrealScript. //! Fermented `UnrealScript`.
//! //!
//! The Fermented dialect extends original UnrealScript with array-size //! Extends original `UnrealScript` by allowing array-size expressions and
//! expressions and declarator initializers. //! declarator initializers.
#![allow(clippy::option_if_let_else)] #![allow(clippy::option_if_let_else)]
use std::ops::ControlFlow; use std::ops::ControlFlow;
use crate::ast::{self, OptionalExpression, VariableDeclarator, VariableDeclaratorRef}; use crate::arena::ArenaVec;
use crate::ast::{OptionalExpression, VariableDeclarator, VariableDeclaratorRef};
use crate::lexer::{Token, TokenPosition, TokenSpan}; use crate::lexer::{Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, SyncLevel}; use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum VariableDeclaratorListParseStep { enum VariableDeclaratorParseState {
ExpectingDeclarator, ExpectingDeclarator,
ExpectingSeparator, ExpectingSeparator,
} }
#[derive(Debug)]
struct VariableDeclaratorListParseState<'src, 'arena> {
declarators: ast::DeclaratorsList<'src, 'arena>,
declaration_keyword_position: TokenPosition,
variable_type_span: TokenSpan,
pending_separator_position: Option<TokenPosition>,
previous_declarator_span: Option<TokenSpan>,
parse_step: VariableDeclaratorListParseStep,
}
impl<'src, 'arena> VariableDeclaratorListParseState<'src, 'arena> {
#[must_use]
fn new(
parser: &Parser<'src, 'arena>,
declaration_keyword_position: TokenPosition,
variable_type_span: TokenSpan,
) -> Self {
Self {
declarators: parser.arena.vec(),
declaration_keyword_position,
variable_type_span,
pending_separator_position: None,
previous_declarator_span: None,
parse_step: VariableDeclaratorListParseStep::ExpectingDeclarator,
}
}
fn set_declarator_expected(&mut self, separator_position: TokenPosition) {
self.pending_separator_position = Some(separator_position);
self.parse_step = VariableDeclaratorListParseStep::ExpectingDeclarator;
}
fn set_separator_expected(&mut self) {
self.pending_separator_position = None;
self.parse_step = VariableDeclaratorListParseStep::ExpectingSeparator;
}
fn push_declarator(&mut self, declarator: VariableDeclaratorRef<'src, 'arena>) {
self.previous_declarator_span = Some(*declarator.span());
self.declarators.push(declarator);
self.set_separator_expected();
}
fn missing_declarator_error_kind(&self) -> ParseErrorKind {
if self.declarators.is_empty() {
ParseErrorKind::VariableDeclaratorListEmpty
} else {
ParseErrorKind::VariableDeclaratorExpected
}
}
}
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a comma-separated variable declarator list after a declaration /// Parses a comma-separated list of variable declarators.
/// keyword and type.
/// ///
/// Accepts optional array-size expressions and `=` initializers. /// Accepts optional array-size expressions and `=` initializers.
///
/// Stops before `;` or before a token that cannot continue the list. The
/// semicolon is left for the caller to consume or diagnose as the enclosing
/// statement terminator.
#[must_use] #[must_use]
pub(crate) fn parse_variable_declarators( pub(crate) fn parse_variable_declarators(
&mut self, &mut self,
declaration_keyword_position: TokenPosition, ) -> ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>> {
variable_type_span: TokenSpan, use VariableDeclaratorParseState::{ExpectingDeclarator, ExpectingSeparator};
) -> ast::DeclaratorsList<'src, 'arena> {
use VariableDeclaratorListParseStep::{ExpectingDeclarator, ExpectingSeparator};
let mut state = VariableDeclaratorListParseState::new( let mut declarators = self.arena.vec();
self, let mut parser_state = ExpectingDeclarator;
declaration_keyword_position,
variable_type_span,
);
// Progress invariant: every branch that continues the loop consumes at
// least one token. Branches that only report an error must stop
// the loop, so the parser remains finite even if
// `ensure_forward_progress` is only a debug/safety check.
while let Some((next_token, next_token_position)) = self.peek_token_and_position() { while let Some((next_token, next_token_position)) = self.peek_token_and_position() {
match (state.parse_step, next_token) { match (parser_state, next_token) {
(ExpectingDeclarator, Token::Semicolon) => { (ExpectingDeclarator, Token::Semicolon) => {
self.report_missing_variable_declarator_at_current_token(&state); self.report_error_here(ParseErrorKind::DeclEmptyVariableDeclarations);
return state.declarators; return declarators;
} }
(ExpectingDeclarator, Token::Comma) => { (ExpectingDeclarator, Token::Comma) => {
if self if self
.recover_empty_variable_declarator_run(next_token_position, &state) .recover_empty_variable_declarator(next_token_position)
.is_break() .is_break()
{ {
return state.declarators; return declarators;
} }
} }
(ExpectingDeclarator, _) if next_token.is_valid_identifier_name() => {
if self
.parse_and_push_variable_declarator(&mut state)
.is_break()
{
break;
}
}
// A token that cannot start a declarator ends the list.
// Report whether the whole list is missing or only the current
// declarator slot is missing, then leave the enclosing
// terminator for the caller to report.
(ExpectingDeclarator, _) => { (ExpectingDeclarator, _) => {
self.report_missing_variable_declarator_at_current_token(&state); if self
.parse_variable_declarator_into(&mut declarators)
.is_break()
{
// Breaking means we've failed to parse declarator
self.report_error_here(ParseErrorKind::DeclEmptyVariableDeclarations);
break; break;
} }
parser_state = ExpectingSeparator;
}
(ExpectingSeparator, Token::Comma) => { (ExpectingSeparator, Token::Comma) => {
self.advance(); // ',' self.advance();
state.set_declarator_expected(next_token_position); parser_state = ExpectingDeclarator;
} }
(ExpectingSeparator, Token::Semicolon) => break, (ExpectingSeparator, Token::Semicolon) => break,
// Missing-separator recovery is intentionally stricter than (ExpectingSeparator, _) => {
// normal declarator parsing, so identifier-like keywords are
// not treated as continuations (no `is_valid_identifier_name`).
(ExpectingSeparator, Token::Identifier) => {
if self if self
.recover_missing_separator_before_variable_declarator( .recover_missing_variable_declarator_separator(
next_token_position, next_token_position,
&mut state, &mut declarators,
) )
.is_break() .is_break()
{ {
break; break;
} }
} }
// Any other invalid cases in place of separator should be
// handled as errors by the caller to avoid error cascade.
(ExpectingSeparator, _) => break,
} }
// Guard against parser bugs that would otherwise leave
// block parsing stuck on the same token.
self.ensure_forward_progress(next_token_position); self.ensure_forward_progress(next_token_position);
} }
// At end-of-file, the caller is expected to report the more relevant // In case of reaching EOF here, it does not matter if we emit
// enclosing error, so this parser does not add another diagnostic. // an additional diagnostic.
state.declarators // The caller is expected to report the more relevant enclosing error.
declarators
} }
/// Reports a missing variable declarator and attaches declaration context. fn recover_empty_variable_declarator(
fn report_missing_variable_declarator_at_current_token(
&mut self,
state: &VariableDeclaratorListParseState<'src, 'arena>,
) {
let error_position = self.peek_position_or_eof();
self.make_error_at(state.missing_declarator_error_kind(), error_position)
.related_token("declaration_keyword", state.declaration_keyword_position)
.related("variable_type", state.variable_type_span)
.maybe_related_token("separator", state.pending_separator_position)
.report(self);
}
/// Recovers from one or more empty declarator slots produced by commas.
///
/// Called when the current token is `,`. Returns [`ControlFlow::Continue`]
/// only when the next remaining token can start a declarator.
fn recover_empty_variable_declarator_run(
&mut self, &mut self,
error_start_position: TokenPosition, error_start_position: TokenPosition,
state: &VariableDeclaratorListParseState<'src, 'arena>,
) -> ControlFlow<()> { ) -> ControlFlow<()> {
let missing_first_declarator = while self.peek_token() == Some(Token::Comma) {
state.declarators.is_empty() && state.pending_separator_position.is_none();
while let Some(Token::Comma) = self.peek_token() {
self.advance(); self.advance();
} }
self.make_error_at_last_consumed(ParseErrorKind::DeclEmptyVariableDeclarations)
let mut error = self
.make_error_at_last_consumed(ParseErrorKind::VariableDeclaratorExpected)
.widen_error_span_from(error_start_position) .widen_error_span_from(error_start_position)
.extend_blame_start_to_covered_start() .report_error(self);
.related_token("declaration_keyword", state.declaration_keyword_position) if matches!(self.peek_token(), Some(Token::Semicolon) | None) {
.related("variable_type", state.variable_type_span)
.maybe_related_token("separator", state.pending_separator_position)
.with_flag("empty_declarator_run");
if missing_first_declarator {
error = error.with_flag("first_declarator");
}
error.report_error(self);
if self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name())
{
ControlFlow::Continue(())
} else {
ControlFlow::Break(()) ControlFlow::Break(())
} else {
ControlFlow::Continue(())
} }
} }
/// Parses one variable declarator and appends it to `state`. fn parse_variable_declarator_into(
///
/// Returns [`ControlFlow::Continue`] only after consuming at least the
/// declarator name and advancing `state` to the separator phase. Returns
/// [`ControlFlow::Break`] on parse failure so the caller does not retry at
/// the same token.
fn parse_and_push_variable_declarator(
&mut self, &mut self,
state: &mut VariableDeclaratorListParseState<'src, 'arena>, declarators: &mut ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
) -> ControlFlow<()> { ) -> ControlFlow<()> {
if let Some(parsed_declarator) = self if let Some(parsed_declarator) = self
.parse_variable_declarator(state.declaration_keyword_position) .parse_variable_declarator()
.sync_error_until(self, SyncLevel::StatementStart) .sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self) .ok_or_report(self)
{ {
state.push_declarator(parsed_declarator); declarators.push(parsed_declarator);
ControlFlow::Continue(()) ControlFlow::Continue(())
} else { } else {
// Keep the fallback so changes in declarator recovery do not turn
// this helper into an unconditional continuation.
ControlFlow::Break(()) ControlFlow::Break(())
} }
} }
/// Recovers from a declarator that starts without a preceding comma. fn recover_missing_variable_declarator_separator(
///
/// Returns [`ControlFlow::Continue`] only after consuming the declarator
/// and reporting a missing-separator diagnostic.
/// Returns [`ControlFlow::Break`] on parse failure so the caller does not
/// retry at the same token.
fn recover_missing_separator_before_variable_declarator(
&mut self, &mut self,
error_start_position: TokenPosition, error_start_position: TokenPosition,
state: &mut VariableDeclaratorListParseState<'src, 'arena>, declarators: &mut ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
) -> ControlFlow<()> { ) -> ControlFlow<()> {
if let Some(parsed_declarator) = self if let Some(parsed_declarator) = self
.parse_variable_declarator(state.declaration_keyword_position) .parse_variable_declarator()
.widen_error_span_from(error_start_position) .widen_error_span_from(error_start_position)
.sync_error_until(self, SyncLevel::StatementStart) .sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self) .ok_or_report(self)
{ {
self.make_error_at_last_consumed( self.make_error_at_last_consumed(ParseErrorKind::DeclNoSeparatorBetweenVariableDeclarations)
ParseErrorKind::VariableDeclaratorListMissingSeparator,
)
.widen_error_span_from(error_start_position) .widen_error_span_from(error_start_position)
.related_token("declaration_keyword", state.declaration_keyword_position)
.maybe_related("last_declarator", state.previous_declarator_span)
.report_error(self); .report_error(self);
state.push_declarator(parsed_declarator); declarators.push(parsed_declarator);
ControlFlow::Continue(()) ControlFlow::Continue(())
} else { } else {
// Keep the fallback so changes in declarator recovery do not turn
// this helper into an unconditional continuation.
ControlFlow::Break(()) ControlFlow::Break(())
} }
} }
/// Parses a single variable declarator.
fn parse_variable_declarator( fn parse_variable_declarator(
&mut self, &mut self,
declaration_keyword_position: TokenPosition,
) -> ParseResult<'src, 'arena, VariableDeclaratorRef<'src, 'arena>> { ) -> ParseResult<'src, 'arena, VariableDeclaratorRef<'src, 'arena>> {
let variable_name = self.parse_identifier(ParseErrorKind::VariableDeclaratorExpected)?; let name = self.parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)?;
let array_size = let array_size = self.parse_optional_array_size();
self.parse_optional_array_size(declaration_keyword_position, variable_name.0); let initializer = self.parse_optional_variable_initializer();
let initializer = self.parse_optional_variable_initializer(variable_name.0); let span = TokenSpan::range(name.0, self.last_consumed_position_or_start());
let declarator_span =
TokenSpan::range(variable_name.0, self.last_consumed_position_or_start());
Ok(self.arena.alloc_node( Ok(self.arena.alloc_node(
VariableDeclarator { VariableDeclarator {
name: variable_name, name,
initializer, initializer,
array_size, array_size,
}, },
declarator_span, span,
)) ))
} }
/// Parses an optional array-size expression. fn parse_optional_array_size(&mut self) -> OptionalExpression<'src, 'arena> {
///
/// Returns [`None`] when no array size is present or when the size
/// expression cannot be recovered.
fn parse_optional_array_size(
&mut self,
declaration_keyword_position: TokenPosition,
variable_name_position: TokenPosition,
) -> OptionalExpression<'src, 'arena> {
if !self.eat(Token::LeftBracket) { if !self.eat(Token::LeftBracket) {
return None; return None;
} }
let left_bracket_position = self.last_consumed_position_or_start(); let array_size_expression = self.parse_expression();
let array_size_expression = self
.parse_required_expression_with_context(
ParseErrorKind::VariableDeclaratorArraySizeExpected,
variable_name_position,
left_bracket_position,
)
.sync_error_until(self, SyncLevel::CloseBracket)
.ok_or_report(self);
self.expect( self.expect(
Token::RightBracket, Token::RightBracket,
ParseErrorKind::VariableDeclaratorArraySizeMissingClosingBracket, ParseErrorKind::DeclExpectedRightBracketAfterArraySize,
) )
.related_token("declaration_keyword", declaration_keyword_position) .sync_error_at(self, SyncLevel::CloseBracket)
.related_token("open_bracket", left_bracket_position)
.sync_error_at_matching_delimiter(self, left_bracket_position)
.report_error(self); .report_error(self);
array_size_expression Some(array_size_expression)
} }
/// Parses an optional initializer. fn parse_optional_variable_initializer(&mut self) -> OptionalExpression<'src, 'arena> {
/// self.eat(Token::Assign).then(|| self.parse_expression())
/// Returns [`None`] both when the initializer is absent and when its
/// expression cannot be recovered. This makes the recovered AST ambiguous,
/// but that is acceptable because callers should treat it only as a
/// best-effort result after a reported syntax error.
fn parse_optional_variable_initializer(
&mut self,
variable_name_position: TokenPosition,
) -> OptionalExpression<'src, 'arena> {
if self.eat(Token::Assign) {
let assignment_operator_position = self.last_consumed_position_or_start();
self.parse_required_expression_with_context(
ParseErrorKind::VariableDeclaratorInitializerExpected,
variable_name_position,
assignment_operator_position,
)
.sync_error_until(self, SyncLevel::ListSeparator)
.ok_or_report(self)
} else {
None
}
} }
} }

4
rottlib/src/parser/grammar/expression/control_flow/mod.rs
View File

@ -1,8 +1,8 @@
//! Control expression parsing for Fermented UnrealScript. //! Control expression parsing for Fermented `UnrealScript`.
//! //!
//! ## Condition boundary recovery and legacy compatibility //! ## Condition boundary recovery and legacy compatibility
//! //!
//! Fermented UnrealScript allows omitting parentheses `(...)` around //! Fermented `UnrealScript` allows omitting parentheses `(...)` around
//! condition expressions of `if`/`while`/`until` and similar constructs. //! condition expressions of `if`/`while`/`until` and similar constructs.
//! Conditions are therefore parsed as ordinary expressions by default. //! Conditions are therefore parsed as ordinary expressions by default.
//! //!

102
rottlib/src/parser/grammar/expression/identifier.rs
View File

@ -1,33 +1,35 @@
//! Identifier parsing for Fermented UnrealScript. //! Identifier parsing for Fermented `UnrealScript`.
//! //!
//! Provides shared routines for parsing plain identifiers and dot-qualified //! Provides shared routines for parsing both regular and qualified identifiers,
//! identifier paths, e.g. `KFChar.ZombieClot`. //! e.g. `KFChar.ZombieClot`.
use crate::arena; use crate::arena::{self, ArenaVec};
use crate::ast::{self, IdentifierToken, QualifiedIdentifierRef}; use crate::ast::{IdentifierToken, QualifiedIdentifier, QualifiedIdentifierRef};
use crate::lexer::{self, Token, TokenSpan}; use crate::lexer::{self, Token, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, diagnostic_labels}; use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt};
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses an identifier. /// Parses an identifier.
/// ///
/// Consumes one identifier token and returns its position. /// On failure (unexpected end-of-file or a token that cannot be used as an
/// Produces `invalid_identifier_error_kind` if the next token is missing or /// identifier), produces `invalid_identifier_error_kind`.
/// cannot be used as an identifier without consuming anything.
pub(crate) fn parse_identifier( pub(crate) fn parse_identifier(
&mut self, &mut self,
invalid_identifier_error_kind: ParseErrorKind, invalid_identifier_error_kind: ParseErrorKind,
) -> ParseResult<'src, 'arena, IdentifierToken> { ) -> ParseResult<'src, 'arena, IdentifierToken> {
let (token, token_position) = let (token, token_position) =
self.require_token_and_position(invalid_identifier_error_kind)?; self.require_token_and_position(invalid_identifier_error_kind)?;
let identifier = Self::identifier_token_from_token(token, token_position) let identifier = Parser::identifier_token_from_token(token, token_position)
.ok_or_else(|| self.make_error_at(invalid_identifier_error_kind, token_position))?; .ok_or_else(|| self.make_error_at_last_consumed(invalid_identifier_error_kind))?;
self.advance(); self.advance();
Ok(identifier) Ok(identifier)
} }
/// Returns an [`IdentifierToken`] if `token` can be used as /// Returns an [`IdentifierToken`] for `token` if it is valid as an
/// an identifier name. /// identifier name.
///
/// This helper performs only token-to-identifier validation/wrapping;
/// it does not consume input from the parser.
pub(crate) fn identifier_token_from_token( pub(crate) fn identifier_token_from_token(
token: Token, token: Token,
token_position: lexer::TokenPosition, token_position: lexer::TokenPosition,
@ -37,74 +39,42 @@ impl<'src, 'arena> Parser<'src, 'arena> {
.then_some(IdentifierToken(token_position)) .then_some(IdentifierToken(token_position))
} }
/// Parses a dot-qualified identifier path. /// Parses a qualified (dot-separated) identifier path,
/// e.g. `KFChar.ZombieClot`.
/// ///
/// Accepts both a single identifier, such as `KFChar`, /// This is used for name paths where each segment must be
/// and a qualified path, such as `KFChar.ZombieClot`. /// a valid identifier and segments are separated by `.` tokens.
/// ///
/// Produces `invalid_identifier_error_kind` when the first identifier or a /// On failure produces an error of specified [`ParseErrorKind`]
/// segment after `.` is missing or invalid. /// `invalid_identifier_error_kind`.
pub(crate) fn parse_qualified_identifier( pub(crate) fn parse_qualified_identifier(
&mut self, &mut self,
invalid_identifier_error_kind: ParseErrorKind, invalid_identifier_error_kind: ParseErrorKind,
) -> ParseResult<'src, 'arena, QualifiedIdentifierRef<'arena>> { ) -> ParseResult<'src, 'arena, QualifiedIdentifierRef<'arena>> {
let head = self.parse_identifier(invalid_identifier_error_kind)?; let head = self.parse_identifier(invalid_identifier_error_kind)?;
let mut tail = None; let mut tail = None;
let mut identifier_span = TokenSpan::new(head.0);
let span_start = head.0;
let mut span_end = span_start;
while let Some((Token::Period, dot_position)) = self.peek_token_and_position() { while let Some((Token::Period, dot_position)) = self.peek_token_and_position() {
self.advance(); // '.' self.advance(); // '.'
let Some((segment_token, segment_position)) = self.peek_token_and_position() else { let next_segment = match self
let next_position = self.peek_position_or_eof(); .parse_identifier(invalid_identifier_error_kind)
return Err(self .widen_error_span_from(head.0)
.make_error_at(invalid_identifier_error_kind, next_position) {
.widen_error_span_from(identifier_span.start) Ok(next_segment) => next_segment,
.related_token(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT, dot_position)); Err(error) => return Err(error.related_token("qualifier_dot", dot_position)),
}; };
let Some(segment) = Self::identifier_token_from_token(segment_token, segment_position) span_end = next_segment.0;
else {
let invalid_segment_position = segment_position; let tail_vec = tail.get_or_insert_with(|| ArenaVec::new_in(self.arena));
let consumed_recovery_tokens = self.recover_invalid_qualified_identifier_tail(); tail_vec.push(next_segment);
let diagnostic_position = if consumed_recovery_tokens {
self.last_consumed_position_or_start()
} else {
invalid_segment_position
};
return Err(self
.make_error_at(invalid_identifier_error_kind, diagnostic_position)
.widen_error_span_from(identifier_span.start)
.blame_token(invalid_segment_position)
.related_token(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT, dot_position));
};
self.advance(); // consume segment
identifier_span.extend_to(segment.0);
let remaining_segments_vec =
tail.get_or_insert_with(|| arena::ArenaVec::new_in(self.arena));
remaining_segments_vec.push(segment);
} }
Ok(arena::ArenaNode::new_in( Ok(arena::ArenaNode::new_in(
ast::QualifiedIdentifier { head, tail }, QualifiedIdentifier { head, tail },
identifier_span, TokenSpan::range(span_start, span_end),
self.arena, self.arena,
)) ))
} }
/// Recovers from an invalid segment in a qualified identifier tail.
///
/// Recovery skips only additional `.segment` suffixes after
/// the invalid token. Returns whether any recovery token was consumed.
fn recover_invalid_qualified_identifier_tail(&mut self) -> bool {
let mut consumed_any = false;
while self.eat(Token::Period) {
consumed_any = true;
if self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name())
{
self.advance();
consumed_any = true;
}
}
consumed_any
}
} }

26
rottlib/src/parser/grammar/expression/literals.rs
View File

@ -1,4 +1,4 @@
//! Literal decoding for Fermented UnrealScript. //! Literal decoding for Fermented `UnrealScript`.
//! //!
//! This module defines the semantic rules for interpreting literal tokens //! This module defines the semantic rules for interpreting literal tokens
//! produced by the lexer. It is responsible only for *decoding* the textual //! produced by the lexer. It is responsible only for *decoding* the textual
@ -7,7 +7,6 @@
//! The rules implemented here intentionally mirror the quirks of //! The rules implemented here intentionally mirror the quirks of
//! Unreal Engine 2’s `UnrealScript`. //! Unreal Engine 2’s `UnrealScript`.
use crate::lexer::TokenPosition;
use crate::parser::{ParseErrorKind, ParseResult}; use crate::parser::{ParseErrorKind, ParseResult};
impl<'src, 'arena> crate::parser::Parser<'src, 'arena> { impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
@ -25,11 +24,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
/// ///
/// On failure, returns [`ParseErrorKind::InvalidNumericLiteral`] at /// On failure, returns [`ParseErrorKind::InvalidNumericLiteral`] at
/// the parser's current cursor position. /// the parser's current cursor position.
pub(crate) fn decode_integer_literal( pub(crate) fn decode_integer_literal(&self, literal: &str) -> ParseResult<'src, 'arena, u128> {
&self,
literal: &str,
literal_position: TokenPosition,
) -> ParseResult<'src, 'arena, u128> {
let (base, content) = match literal.split_at_checked(2) { let (base, content) = match literal.split_at_checked(2) {
Some(("0b" | "0B", stripped)) => (2, stripped), Some(("0b" | "0B", stripped)) => (2, stripped),
Some(("0o" | "0O", stripped)) => (8, stripped), Some(("0o" | "0O", stripped)) => (8, stripped),
@ -37,9 +32,8 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
_ => (10, literal), _ => (10, literal),
}; };
let digits_without_underscores = content.replace('_', ""); let digits_without_underscores = content.replace('_', "");
u128::from_str_radix(&digits_without_underscores, base).map_err(|_| { u128::from_str_radix(&digits_without_underscores, base)
self.make_error_at(ParseErrorKind::InvalidNumericLiteral, literal_position) .map_err(|_| self.make_error_at_last_consumed(ParseErrorKind::InvalidNumericLiteral))
})
} }
/// Decodes a float literal as `f64`, following the permissive and only /// Decodes a float literal as `f64`, following the permissive and only
@ -72,11 +66,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
/// On failure, this function returns /// On failure, this function returns
/// [`ParseErrorKind::InvalidNumericLiteral`] at the current parser /// [`ParseErrorKind::InvalidNumericLiteral`] at the current parser
/// position. /// position.
pub(crate) fn decode_float_literal( pub(crate) fn decode_float_literal(&self, literal: &str) -> ParseResult<'src, 'arena, f64> {
&self,
literal: &str,
literal_position: TokenPosition,
) -> ParseResult<'src, 'arena, f64> {
let content = literal let content = literal
.strip_suffix('f') .strip_suffix('f')
.or_else(|| literal.strip_suffix('F')) .or_else(|| literal.strip_suffix('F'))
@ -87,9 +77,9 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
.nth(1) .nth(1)
.and_then(|(period_index, _)| content.get(..period_index)) .and_then(|(period_index, _)| content.get(..period_index))
.unwrap_or(content); .unwrap_or(content);
content.parse::<f64>().map_err(|_| { content
self.make_error_at(ParseErrorKind::InvalidNumericLiteral, literal_position) .parse::<f64>()
}) .map_err(|_| self.make_error_at_last_consumed(ParseErrorKind::InvalidNumericLiteral))
} }
/// Unescapes a tokenized string literal into an arena string. /// Unescapes a tokenized string literal into an arena string.

2
rottlib/src/parser/grammar/expression/mod.rs
View File

@ -1,4 +1,4 @@
//! Expression parsing for Fermented UnrealScript. //! Expression parsing for Fermented `UnrealScript`.
//! //!
//! This module group implements the language's expression parser around a //! This module group implements the language's expression parser around a
//! Pratt-style core. It is split into small submodules by role: precedence, //! Pratt-style core. It is split into small submodules by role: precedence,

60
rottlib/src/parser/grammar/expression/pratt.rs
View File

@ -1,4 +1,4 @@
//! Core of the expression parser for Fermented UnrealScript. //! Core of the expression parser for Fermented `UnrealScript`.
//! //!
//! This module implements a Pratt-style parser for the language's expression //! This module implements a Pratt-style parser for the language's expression
//! grammar, supporting: //! grammar, supporting:
@ -13,7 +13,7 @@
//! operators bind. Infix parsing uses the pair of binding powers returned by //! operators bind. Infix parsing uses the pair of binding powers returned by
//! [`super::precedence::infix_precedence_ranks`] to encode associativity. //! [`super::precedence::infix_precedence_ranks`] to encode associativity.
//! The parser infrastructure supports both left- and right-associative //! The parser infrastructure supports both left- and right-associative
//! operators, but Fermented UnrealScript currently defines only //! operators, but Fermented `UnrealScript` currently defines only
//! left-associative ones. //! left-associative ones.
//! //!
//! ## Postfix operator vs "selectors" //! ## Postfix operator vs "selectors"
@ -35,7 +35,7 @@
use crate::ast::{self, Expression, ExpressionRef}; use crate::ast::{self, Expression, ExpressionRef};
use crate::lexer::TokenPosition; use crate::lexer::TokenPosition;
use crate::parser::{ use crate::parser::{
ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, SyncLevel, diagnostic_labels, self, ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, diagnostic_labels,
}; };
pub use super::precedence::PrecedenceRank; pub use super::precedence::PrecedenceRank;
@ -59,30 +59,34 @@ fn forbids_postfix_operators(expression: &ExpressionRef<'_, '_>) -> bool {
} }
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
// TODO: success here guaranees progress
/// Parses an expression. /// Parses an expression.
/// ///
/// Always returns some expression node; any syntax errors are reported /// Always returns some expression node; any syntax errors are reported
/// through the parser's diagnostics. /// through the parser's diagnostics.
/// Returning non-erroneous node guarantees that parser has advanced
/// at least one token.
#[must_use] #[must_use]
pub fn parse_expression(&mut self) -> ExpressionRef<'src, 'arena> { pub fn parse_expression(&mut self) -> ExpressionRef<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST) self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
.sync_error_until(self, SyncLevel::ExpressionStart) .sync_error_until(self, parser::SyncLevel::ExpressionStart)
.unwrap_or_fallback(self) .unwrap_or_fallback(self)
} }
/// Parses an expression in a grammar position where an expression is /// Parses an expression in a grammar position where an expression is
/// required. /// required.
/// ///
/// If the next token cannot start an expression, this reports /// This is the checked variant of [`Parser::parse_expression`]. If the next
/// `bad_start_error_kind` at that token, starts expression-start recovery /// token is known not to be a valid expression starter, this reports
/// and returns an error expression result. /// `bad_start_error_kind`, consumes the bad token, and starts panic-mode
/// recovery until [`crate::parser::SyncLevel::ExpressionStart`].
/// ///
/// `required_by_position` identifies the token or construct that created /// `required_by_position` identifies the token or construct that created
/// the requirement for an expression. It is attached to the diagnostic with /// the requirement for an expression. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_REQUIRED_BY`] label. /// the [`diagnostic_labels::EXPRESSION_REQUIRED_BY`] label.
pub(crate) fn parse_required_expression( ///
/// `expression_context_position` identifies the local syntactic anchor after
/// which the expression was expected. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_EXPECTED_AFTER`] label.
pub(super) fn parse_required_expression(
&mut self, &mut self,
bad_start_error_kind: ParseErrorKind, bad_start_error_kind: ParseErrorKind,
required_by_position: TokenPosition, required_by_position: TokenPosition,
@ -92,7 +96,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return Err(self return Err(self
.make_error_at(bad_start_error_kind, error_position) .make_error_at(bad_start_error_kind, error_position)
.sync_error_until(self, SyncLevel::ExpressionStart) .sync_error_until(self, crate::parser::SyncLevel::ExpressionStart)
.blame_token(error_position) .blame_token(error_position)
.related_token( .related_token(
diagnostic_labels::EXPRESSION_REQUIRED_BY, diagnostic_labels::EXPRESSION_REQUIRED_BY,
@ -103,22 +107,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST) self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
} }
/// Parses an expression in a grammar position where an expression is pub(super) fn parse_required_expression_with_context(
/// required, and records the local syntactic context where it was expected.
///
/// If the next token cannot start an expression, this reports
/// `bad_start_error_kind` at that token, starts expression-start recovery,
/// and returns an error expression result.
///
/// `required_by_position` identifies the token or construct that created
/// the requirement for an expression. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_REQUIRED_BY`] label.
///
/// `expression_context_position` identifies the local syntactic anchor
/// after which the expression was expected. It is attached to
/// the diagnostic with the [`diagnostic_labels::EXPRESSION_EXPECTED_AFTER`]
/// label.
pub(crate) fn parse_required_expression_with_context(
&mut self, &mut self,
bad_start_error_kind: ParseErrorKind, bad_start_error_kind: ParseErrorKind,
required_by_position: TokenPosition, required_by_position: TokenPosition,
@ -129,7 +118,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return Err(self return Err(self
.make_error_at(bad_start_error_kind, error_position) .make_error_at(bad_start_error_kind, error_position)
.sync_error_until(self, SyncLevel::ExpressionStart) .sync_error_until(self, crate::parser::SyncLevel::ExpressionStart)
.blame_token(error_position) .blame_token(error_position)
.related_token( .related_token(
diagnostic_labels::EXPRESSION_REQUIRED_BY, diagnostic_labels::EXPRESSION_REQUIRED_BY,
@ -144,11 +133,6 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST) self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
} }
/// Creates an error expression at `position`.
///
/// The returned node has a single-token span anchored at `position` and
/// can be used as a fallback expression when parsing cannot produce
/// a valid one.
pub(super) fn make_error_expression_at( pub(super) fn make_error_expression_at(
&self, &self,
position: TokenPosition, position: TokenPosition,
@ -165,7 +149,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
fn parse_expression_with_min_precedence_rank( fn parse_expression_with_min_precedence_rank(
&mut self, &mut self,
min_precedence_rank: PrecedenceRank, min_precedence_rank: PrecedenceRank,
) -> ParseExpressionResult<'src, 'arena> { ) -> parser::ParseExpressionResult<'src, 'arena> {
let mut left_hand_side = self.parse_prefix_or_primary()?; let mut left_hand_side = self.parse_prefix_or_primary()?;
left_hand_side = self.parse_selectors_after(left_hand_side)?; left_hand_side = self.parse_selectors_after(left_hand_side)?;
// We disallow only postfix operators after expression forms that // We disallow only postfix operators after expression forms that
@ -190,7 +174,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a prefix or primary expression (Pratt parser's "nud" or /// Parses a prefix or primary expression (Pratt parser's "nud" or
/// null denotation). /// null denotation).
fn parse_prefix_or_primary(&mut self) -> ParseExpressionResult<'src, 'arena> { fn parse_prefix_or_primary(&mut self) -> parser::ParseExpressionResult<'src, 'arena> {
let (token, token_lexeme, token_position) = let (token, token_lexeme, token_position) =
self.require_token_lexeme_and_position(ParseErrorKind::ExpressionExpected)?; self.require_token_lexeme_and_position(ParseErrorKind::ExpressionExpected)?;
// Avoid advancing over an obviously wrong token; // Avoid advancing over an obviously wrong token;
@ -242,7 +226,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
mut left_hand_side: ExpressionRef<'src, 'arena>, mut left_hand_side: ExpressionRef<'src, 'arena>,
min_precedence_rank: PrecedenceRank, min_precedence_rank: PrecedenceRank,
) -> ParseExpressionResult<'src, 'arena> { ) -> parser::ParseExpressionResult<'src, 'arena> {
while let Some((operator, right_precedence_rank)) = while let Some((operator, right_precedence_rank)) =
self.peek_infix_with_min_precedence_rank(min_precedence_rank) self.peek_infix_with_min_precedence_rank(min_precedence_rank)
{ {
@ -265,7 +249,9 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self, &mut self,
) -> Option<(ast::PostfixOperator, crate::lexer::TokenPosition)> { ) -> Option<(ast::PostfixOperator, crate::lexer::TokenPosition)> {
let (token, token_position) = self.peek_token_and_position()?; let (token, token_position) = self.peek_token_and_position()?;
let operator = ast::PostfixOperator::try_from(token).ok()?; let Ok(operator) = ast::PostfixOperator::try_from(token) else {
return None;
};
Some((operator, token_position)) Some((operator, token_position))
} }

2
rottlib/src/parser/grammar/expression/precedence.rs
View File

@ -1,4 +1,4 @@
//! Precedence tables for Fermented UnrealScript operators. //! Precedence tables for Fermented `UnrealScript` operators.
//! //!
//! These values don't follow the usual *binding power* convention for //! These values don't follow the usual *binding power* convention for
//! a Pratt parser, where tighter binding corresponds to a larger number.\ //! a Pratt parser, where tighter binding corresponds to a larger number.\

102
rottlib/src/parser/grammar/expression/primary/mod.rs
View File

@ -54,20 +54,12 @@
//! `(`, `switch` is parsed as a `switch` expression; otherwise it remains //! `(`, `switch` is parsed as a `switch` expression; otherwise it remains
//! available as an identifier. //! available as an identifier.
use crate::ast::{Expression, ExpressionRef, OptionalExpression, QualifiedIdentifierRef}; use crate::ast::{Expression, ExpressionRef, OptionalExpression};
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan}; use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ use crate::parser::{ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, SyncLevel};
ParseError, ParseErrorKind, ParseExpressionResult, ParseResult, Parser, ResultRecoveryExt,
SyncLevel,
};
mod new; mod new;
pub(crate) struct ParsedClassTypeArgument<'arena> {
pub(crate) type_name: QualifiedIdentifierRef<'arena>,
pub(crate) right_angle_bracket_position: TokenPosition,
}
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a primary expression starting from the provided token. /// Parses a primary expression starting from the provided token.
/// ///
@ -92,12 +84,12 @@ impl<'src, 'arena> Parser<'src, 'arena> {
) -> ParseExpressionResult<'src, 'arena> { ) -> ParseExpressionResult<'src, 'arena> {
Ok(match token { Ok(match token {
Token::IntegerLiteral => { Token::IntegerLiteral => {
let value = self.decode_integer_literal(token_lexeme, token_position)?; let value = self.decode_integer_literal(token_lexeme)?;
self.arena self.arena
.alloc_node_at(Expression::Integer(value), token_position) .alloc_node_at(Expression::Integer(value), token_position)
} }
Token::FloatLiteral => { Token::FloatLiteral => {
let value = self.decode_float_literal(token_lexeme, token_position)?; let value = self.decode_float_literal(token_lexeme)?;
self.arena self.arena
.alloc_node_at(Expression::Float(value), token_position) .alloc_node_at(Expression::Float(value), token_position)
} }
@ -155,8 +147,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
// These keywords remain valid identifiers unless the following // These keywords remain valid identifiers unless the following
// tokens commit to the keyword-led form. // tokens commit to the keyword-led form.
Keyword::For Keyword::For
if let Some(left_parenthesis_position) = if let Some(left_parenthesis_position) = self.peek_for_loop_header_left_parenthesis_position() =>
self.peek_for_loop_header_left_parenthesis_position() =>
{ {
self.advance(); // `(` self.advance(); // `(`
self.parse_for_tail(token_position, left_parenthesis_position) self.parse_for_tail(token_position, left_parenthesis_position)
@ -256,98 +247,77 @@ impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a class type expression of the form `class<...>`. /// Parses a class type expression of the form `class<...>`.
/// ///
/// Assumes the `class` keyword and following `<` token have already been /// Assumes the `class` keyword and following '<' token have already been
/// consumed. Reports and recovers from malformed type syntax locally. /// consumed. Reports and recovers from malformed type syntax locally.
fn parse_class_type_tail( fn parse_class_type_tail(
&mut self, &mut self,
class_keyword_position: TokenPosition, class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition, left_angle_bracket_position: TokenPosition,
) -> ExpressionRef<'src, 'arena> { ) -> ExpressionRef<'src, 'arena> {
let parsed = match self
.parse_class_type_argument_tail(class_keyword_position, left_angle_bracket_position)
{
Ok(parsed) => parsed,
Err(error) => return self.report_error_with_fallback(error),
};
self.arena.alloc_node_between(
Expression::ClassType(parsed.type_name),
class_keyword_position,
parsed.right_angle_bracket_position,
)
}
/// Parses a class type argument of the form `...>`.
///
/// Assumes the `class` keyword and following `<` token have already been
/// consumed. Returns the parsed type argument and closing `>` position.
pub(crate) fn parse_class_type_argument_tail(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
) -> ParseResult<'src, 'arena, ParsedClassTypeArgument<'arena>> {
match self.peek_token_and_position() { match self.peek_token_and_position() {
Some((Token::Greater, right_angle_bracket_position)) => { Some((Token::Greater, right_angle_bracket_position)) => self
self.advance(); .report_missing_class_type_argument(
Err(self.make_missing_class_type_argument_error(
class_keyword_position, class_keyword_position,
left_angle_bracket_position, left_angle_bracket_position,
right_angle_bracket_position, right_angle_bracket_position,
)) ),
}
Some((first_token, _)) if first_token.is_valid_identifier_name() => self Some((first_token, _)) if first_token.is_valid_identifier_name() => self
.parse_nonempty_class_type_argument_tail( .parse_nonempty_class_type_tail(
class_keyword_position, class_keyword_position,
left_angle_bracket_position, left_angle_bracket_position,
), ),
Some((_, bad_position)) => Err(self.make_invalid_class_type_start_error( Some((_, bad_position)) => self.report_invalid_class_type_start(
class_keyword_position, class_keyword_position,
left_angle_bracket_position, left_angle_bracket_position,
bad_position, bad_position,
)), ),
None => Err(self.make_invalid_class_type_start_error( None => self.report_invalid_class_type_start(
class_keyword_position, class_keyword_position,
left_angle_bracket_position, left_angle_bracket_position,
self.file.eof(), self.file.eof(),
)), ),
} }
} }
/// Parses a nonempty class type argument followed by its closing `>`. fn parse_nonempty_class_type_tail(
///
/// Assumes the next token can start a qualified type name.
fn parse_nonempty_class_type_argument_tail(
&mut self, &mut self,
class_keyword_position: TokenPosition, class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition, left_angle_bracket_position: TokenPosition,
) -> ParseResult<'src, 'arena, ParsedClassTypeArgument<'arena>> { ) -> ExpressionRef<'src, 'arena> {
let type_name = self let class_type = match self
.parse_qualified_identifier(ParseErrorKind::ClassTypeExpectedQualifiedTypeName) .parse_qualified_identifier(ParseErrorKind::ClassTypeExpectedQualifiedTypeName)
.widen_error_span_from(class_keyword_position) .widen_error_span_from(class_keyword_position)
.extend_blame_to_next_token(self)
.sync_error_at(self, SyncLevel::CloseAngleBracket) .sync_error_at(self, SyncLevel::CloseAngleBracket)
.related_token("class_keyword", class_keyword_position)?; .related_token("class_keyword", class_keyword_position)
{
Ok(class_type) => class_type,
Err(error) => return self.report_error_with_fallback(error),
};
let right_angle_bracket_position = self let right_angle_bracket_position = self
.expect( .expect(
Token::Greater, Token::Greater,
ParseErrorKind::ClassTypeMissingClosingAngleBracket, ParseErrorKind::ClassTypeMissingClosingAngleBracket,
) )
.widen_error_span_from(class_keyword_position) .widen_error_span_from(class_keyword_position)
.sync_error_at(self, SyncLevel::CloseAngleBracket)
.related_token("left_angle_bracket", left_angle_bracket_position) .related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position)?; .related_token("class_keyword", class_keyword_position)
.unwrap_or_fallback(self);
Ok(ParsedClassTypeArgument { self.arena.alloc_node_between(
type_name, Expression::ClassType(class_type),
class_keyword_position,
right_angle_bracket_position, right_angle_bracket_position,
}) )
} }
fn make_missing_class_type_argument_error( fn report_missing_class_type_argument(
&mut self, &mut self,
class_keyword_position: TokenPosition, class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition, left_angle_bracket_position: TokenPosition,
right_angle_bracket_position: TokenPosition, right_angle_bracket_position: TokenPosition,
) -> ParseError { ) -> ExpressionRef<'src, 'arena> {
self.advance();
self.make_error_at_last_consumed(ParseErrorKind::ClassTypeMissingTypeArgument) self.make_error_at_last_consumed(ParseErrorKind::ClassTypeMissingTypeArgument)
.widen_error_span_from(class_keyword_position) .widen_error_span_from(class_keyword_position)
.blame(TokenSpan::range( .blame(TokenSpan::range(
@ -356,20 +326,22 @@ impl<'src, 'arena> Parser<'src, 'arena> {
)) ))
.related_token("left_angle_bracket", left_angle_bracket_position) .related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position) .related_token("class_keyword", class_keyword_position)
.fallback(self)
} }
fn make_invalid_class_type_start_error( fn report_invalid_class_type_start(
&mut self, &mut self,
class_keyword_position: TokenPosition, class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition, left_angle_bracket_position: TokenPosition,
bad_position: TokenPosition, bad_position: TokenPosition,
) -> ParseError { ) -> ExpressionRef<'src, 'arena> {
self.make_error_at_last_consumed(ParseErrorKind::ClassTypeInvalidStart) self.make_error_at_last_consumed(ParseErrorKind::ClassTypeInvalidStart)
.widen_error_span_from(class_keyword_position) .widen_error_span_from(class_keyword_position)
.sync_error_at(self, SyncLevel::CloseAngleBracket) .sync_error_at(self, SyncLevel::CloseAngleBracket)
.blame_token(bad_position) .blame_token(bad_position)
.related_token("left_angle_bracket", left_angle_bracket_position) .related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position) .related_token("class_keyword", class_keyword_position)
.fallback(self)
} }
/// Returns `true` iff the next token is definitely not a valid start of an /// Returns `true` iff the next token is definitely not a valid start of an

6
rottlib/src/parser/grammar/expression/selectors.rs
View File

@ -45,11 +45,7 @@ pub(super) enum ParsedArgumentSlot<'src, 'arena> {
impl<'src, 'arena> Parser<'src, 'arena> { impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses zero or more postfix selectors after `left_hand_side`. /// Parses zero or more postfix selectors after `left_hand_side`.
/// ///
/// If the next token is not a selector starter, consumes nothing and /// Returns the resulting expression after all contiguous selectors.
/// returns `left_hand_side` unchanged.
///
/// Each accepted selector consumes at least its leading token (`.`, `[` or
/// `(`) before selector-specific parsing or recovery is attempted.
pub(super) fn parse_selectors_after( pub(super) fn parse_selectors_after(
&mut self, &mut self,
mut left_hand_side: ExpressionRef<'src, 'arena>, mut left_hand_side: ExpressionRef<'src, 'arena>,

8
rottlib/src/parser/grammar/function/definition.rs
View File

@ -1,4 +1,4 @@
//! Parsing of callable definitions for Fermented UnrealScript //! Parsing of callable definitions for Fermented `UnrealScript`
//! (functions, events, delegates, operators). //! (functions, events, delegates, operators).
use crate::arena::ArenaVec; use crate::arena::ArenaVec;
@ -239,10 +239,10 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return None; return None;
} }
let value = match self.peek_token_lexeme_and_position() { let value = match self.peek_token_and_lexeme() {
Some((Token::IntegerLiteral, lex, token_position)) => { Some((Token::IntegerLiteral, lex)) => {
self.advance(); self.advance();
self.decode_integer_literal(lex, token_position).ok_or_report(self) self.decode_integer_literal(lex).ok_or_report(self)
} }
Some(_) => { Some(_) => {
self.report_error_here(ParseErrorKind::OperatorPrecedenceNotIntegerLiteral); self.report_error_here(ParseErrorKind::OperatorPrecedenceNotIntegerLiteral);

2
rottlib/src/parser/grammar/function/lookahead.rs
View File

@ -1,4 +1,4 @@
//! Lookahead for callable headers in Fermented UnrealScript. //! Lookahead for callable headers in Fermented `UnrealScript`.
use crate::lexer::{Keyword, Token}; use crate::lexer::{Keyword, Token};
use crate::parser::Parser; use crate::parser::Parser;

76
rottlib/src/parser/grammar/statement.rs
View File

@ -1,7 +1,7 @@
//! Statement parsing for the language front-end. //! Statement parsing for the language front-end.
//! //!
//! Implements a simple recursive-descent parser for //! Implements a simple recursive-descent parser for
//! *Fermented UnrealScript statements*. //! *Fermented `UnrealScript` statements*.
use crate::ast::{Statement, StatementRef}; use crate::ast::{Statement, StatementRef};
use crate::lexer::{Keyword, Token, TokenSpan}; use crate::lexer::{Keyword, Token, TokenSpan};
@ -13,13 +13,14 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
/// Does not consume a trailing `;` except for [`Statement::Empty`]. /// Does not consume a trailing `;` except for [`Statement::Empty`].
/// The caller handles semicolons (WRONG NOW - WE MUST HANDLE THEM). Returns [`Some`] if a statement is /// The caller handles semicolons (WRONG NOW - WE MUST HANDLE THEM). Returns [`Some`] if a statement is
/// recognized; otherwise [`None`]. /// recognized; otherwise [`None`].
/// ALSO WE SPECIFICALLY DONT HANDLE EXPRESSION TYPE STATEMENTS - on them also return NONE!!!! /// ALSO WE SPECIFICALLY DONT HANDLE EXPRESSION TYPE STATEMENTS
/// Basically only generates errors if statement is recognized, but got fucked up.
#[must_use] #[must_use]
pub(crate) fn parse_statement(&mut self) -> Option<StatementRef<'src, 'arena>> { pub(crate) fn parse_statement(&mut self) -> Option<StatementRef<'src, 'arena>> {
let Some((token, lexeme, position)) = self.peek_token_lexeme_and_position() else { let Some((token, lexeme, position)) = self.peek_token_lexeme_and_position() else {
self.report_error_here(ParseErrorKind::UnexpectedEndOfFile);
return None; return None;
}; };
match token { match token {
// Empty statement // Empty statement
Token::Semicolon => { Token::Semicolon => {
@ -29,41 +30,17 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
.alloc_node(Statement::Empty, TokenSpan::new(position)), .alloc_node(Statement::Empty, TokenSpan::new(position)),
) )
} }
// UnrealScript `local` declaration // UnrealScript `local` declaration
Token::Keyword(Keyword::Local) => { Token::Keyword(Keyword::Local) => {
self.advance(); // `local` self.advance(); // `local`
let start = position; let start = position;
let Some(type_spec) = self
.parse_type_specifier() let type_spec = self.parse_type_specifier().unwrap_or_fallback(self);
.sync_error_at(self, crate::parser::SyncLevel::StatementTerminator) let declarators = self.parse_variable_declarators();
.ok_or_report(self) // TODO: parse
else {
return Some(crate::arena::ArenaNode::new_in( let span = TokenSpan::range(start, self.last_consumed_position_or_start());
Statement::Error,
crate::lexer::TokenSpan::new(position),
self.arena,
));
};
let declarators = self.parse_variable_declarators(position, *type_spec.span());
let mut span = TokenSpan::range(start, self.last_consumed_position_or_start());
let mut error = if self.peek_token().is_some() {
self
.expect(
Token::Semicolon,
ParseErrorKind::LocalVariableDeclarationMissingSemicolon,
)
.widen_error_span_from(position)
.related_token("local_keyword", position)
} else {
Ok(self.peek_position_or_eof())
};
if let Some(last_declarator) = declarators.last() {
error = error.related("last_declarator", *last_declarator.span())
}
let semicolon_position = error.ok_or_report(self);
if let Some(semicolon_position) = semicolon_position {
span.extend_to(semicolon_position);
}
Some(self.arena.alloc_node( Some(self.arena.alloc_node(
Statement::LocalVariableDeclaration { Statement::LocalVariableDeclaration {
type_spec, type_spec,
@ -72,6 +49,8 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
span, span,
)) ))
} }
// Label: Ident ':' (also tolerate Begin:/End:)
Token::Identifier | Token::Keyword(Keyword::Begin | Keyword::End) Token::Identifier | Token::Keyword(Keyword::Begin | Keyword::End)
if matches!(self.peek_token_at(1), Some(Token::Colon)) => if matches!(self.peek_token_at(1), Some(Token::Colon)) =>
{ {
@ -82,23 +61,26 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
TokenSpan::range(position, self.last_consumed_position_or_start()), TokenSpan::range(position, self.last_consumed_position_or_start()),
)) ))
} }
// Nested function/event/operator inside blocks // Nested function/event/operator inside blocks
_ if is_function_starter(token) => { t if t == Token::Keyword(Keyword::Function)
let callable_definition = self.parse_callable_definition(); || t == Token::Keyword(Keyword::Event)
let span = *callable_definition.span(); || t.is_valid_function_modifier() =>
Some( {
self.arena let f = self.parse_callable_definition();
.alloc_node(Statement::Function(callable_definition), span),
) let span = *f.span();
Some(self.arena.alloc_node(Statement::Function(f), span))
} }
// C-like variable declaration starting with a TypeSpec
/*token if self.looks_like_variable_declaration_start(token) => Some(
self.parse_variable_declaration_start()
.sync_error_until(self, SyncLevel::Statement)
.unwrap_or_fallback(self),
),*/
// Not a statement // Not a statement
_ => None, _ => None,
} }
} }
} }
fn is_function_starter(token: Token) -> bool {
token.is_valid_function_modifier()
|| token == Token::Keyword(Keyword::Function)
|| token == Token::Keyword(Keyword::Event)
}

3
rottlib/src/parser/mod.rs
View File

@ -1,4 +1,4 @@
//! Parser for Fermented UnrealScript (`FerUS`). //! Parser for Fermented `UnrealScript` (`FerUS`).
//! //!
//! Consumes tokens from [`crate::lexer::TokenizedFile`] and allocates AST //! Consumes tokens from [`crate::lexer::TokenizedFile`] and allocates AST
//! nodes in [`crate::arena::Arena`]. Basic expressions use a Pratt parser; //! nodes in [`crate::arena::Arena`]. Basic expressions use a Pratt parser;
@ -46,7 +46,6 @@ pub type ParseExpressionResult<'src, 'arena> =
pub(crate) mod diagnostic_labels { pub(crate) mod diagnostic_labels {
pub(crate) const EXPRESSION_REQUIRED_BY: &str = "expression_required_by"; pub(crate) const EXPRESSION_REQUIRED_BY: &str = "expression_required_by";
pub(crate) const EXPRESSION_EXPECTED_AFTER: &str = "expression_expected_after"; pub(crate) const EXPRESSION_EXPECTED_AFTER: &str = "expression_expected_after";
pub(crate) const QUALIFIED_IDENTIFIER_DOT: &str = "qualifier_dot";
} }
// TODO: add some kind of bailing for infinite loops // TODO: add some kind of bailing for infinite loops

32
rottlib/src/parser/recovery.rs
View File

@ -49,7 +49,7 @@ pub enum SyncLevel {
/// A statement boundary or statement starter. /// A statement boundary or statement starter.
/// ///
/// Includes keywords that begin standalone statements / /// Includes `;` and keywords that begin standalone statements /
/// statement-like control-flow forms. /// statement-like control-flow forms.
StatementStart, StatementStart,
@ -333,7 +333,6 @@ pub trait ResultRecoveryExt<'src, 'arena, T>: Sized {
fn blame(self, blame_span: TokenSpan) -> Self; fn blame(self, blame_span: TokenSpan) -> Self;
fn related(self, tag: impl Into<String>, related_span: TokenSpan) -> Self; fn related(self, tag: impl Into<String>, related_span: TokenSpan) -> Self;
fn with_flag(self, tag: impl Into<String>) -> Self;
fn blame_token(self, blame_position: TokenPosition) -> Self { fn blame_token(self, blame_position: TokenPosition) -> Self {
self.blame(TokenSpan::new(blame_position)) self.blame(TokenSpan::new(blame_position))
@ -350,26 +349,6 @@ pub trait ResultRecoveryExt<'src, 'arena, T>: Sized {
self.related(tag, TokenSpan::new(related_position)) self.related(tag, TokenSpan::new(related_position))
} }
fn maybe_related(self, tag: impl Into<String>, related_span: Option<TokenSpan>) -> Self {
if let Some(related_span) = related_span {
self.related(tag, related_span)
} else {
self
}
}
fn maybe_related_token(
self,
tag: impl Into<String>,
related_position: Option<TokenPosition>,
) -> Self {
if let Some(related_position) = related_position {
self.related(tag, TokenSpan::new(related_position))
} else {
self
}
}
/// Extends the right end of the error span up to but not including /// Extends the right end of the error span up to but not including
/// the next token of the given sync `level`. /// the next token of the given sync `level`.
/// ///
@ -458,10 +437,6 @@ impl<'src, 'arena, T> ResultRecoveryExt<'src, 'arena, T> for ParseResult<'src, '
self.map_err(|error| error.related(tag, related_span)) self.map_err(|error| error.related(tag, related_span))
} }
fn with_flag(self, tag: impl Into<String>) -> Self {
self.map_err(|error| error.with_flag(tag))
}
fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self { fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self {
if let Err(ref mut error) = self { if let Err(ref mut error) = self {
parser.recover_until(level); parser.recover_until(level);
@ -580,11 +555,6 @@ impl<'src, 'arena> ResultRecoveryExt<'src, 'arena, ()> for ParseError {
self self
} }
fn with_flag(mut self, tag: impl Into<String>) -> Self {
self.flags.insert(tag.into());
self
}
fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self { fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self {
parser.recover_until(level); parser.recover_until(level);
self.covered_span.end = std::cmp::max( self.covered_span.end = std::cmp::max(

1031
rottlib/tests/parser_diagnostics/declarations.rs
View File

File diff suppressed because it is too large Load Diff

1
rottlib/tests/parser_diagnostics/mod.rs
View File

@ -7,7 +7,6 @@ use rottlib::parser::Parser;
mod block_items; mod block_items;
mod control_flow_expressions; mod control_flow_expressions;
mod declarations;
mod primary_expressions; mod primary_expressions;
mod selector_expressions; mod selector_expressions;
mod switch_expressions; mod switch_expressions;