NoobsGoHOE/rott
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add delimeter matching to lexer

Browse source
This commit is contained in:
dkanus 2026-04-29 13:50:45 +07:00
commit 9d3313995e
11 changed files with 1196 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

13
dev_tests/src/pretty.rs
View file

@ -1,13 +0,0 @@
// diagnostics_render.rs
use rottlib::diagnostics::{Diagnostic};
use rottlib::lexer::TokenizedFile;
pub fn render_diagnostic(
diag: &Diagnostic,
_file: &TokenizedFile,
file_name: Option<&str>,
colors: bool,
) {
diag.render(_file, file_name.unwrap_or("<default>"));
}

92
dev_tests/src/uc_lexer_verify.rs
View file

@ -19,8 +19,6 @@ use rottlib::diagnostics::Diagnostic as Diag;
use rottlib::lexer::TokenizedFile;
use rottlib::parser::Parser;
mod pretty;
// ---------- CONFIG ----------
const FILE_LIMIT: usize = 10000; // cap on files scanned
const DIAG_SHOW_FIRST: usize = 12; // show first N diagnostics
@ -35,6 +33,43 @@ const ALSO_PRINT_DEBUG_AFTER_PRETTY: bool = true;
// chardet = "0.2"
// encoding_rs = "0.8"
fn render_diagnostic(diag: &Diag, file: &TokenizedFile<'_>, file_name: &str) {
diag.render(file, file_name);
}
fn render_diagnostics_window(diags: &[Diag], tf: &TokenizedFile<'_>, file_name: &str) {
let total = diags.len();
let first_n = DIAG_SHOW_FIRST.min(total);
let last_n = DIAG_SHOW_LAST.min(total.saturating_sub(first_n));
if total > first_n + last_n {
for (k, d) in diags.iter().take(first_n).enumerate() {
render_diagnostic(d, tf, file_name);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{}: {:#?}", k + 1, d);
}
}
eprintln!("... {} diagnostics omitted ...", total - (first_n + last_n));
let start = total - last_n;
for (offset, d) in diags.iter().skip(start).enumerate() {
let idx_global = start + offset + 1;
render_diagnostic(d, tf, file_name);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{idx_global}: {d:#?}");
}
}
} else {
for (k, d) in diags.iter().enumerate() {
render_diagnostic(d, tf, file_name);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{}: {:#?}", k + 1, d);
}
}
}
}
// Linux-only accurate RSS in MB. Fallback uses sysinfo.
fn rss_mb() -> u64 {
#[cfg(target_os = "linux")]
@ -230,11 +265,24 @@ fn main() {
mark("after_tokenize", t0);
// If tokenization error: wait, dump tokens for the first failing file, then exit.
// If tokenization error: wait, print lexer diagnostics for the first failing file, then exit.
if let Some(idx) = tk_error_idx {
let (bad_path, _) = &tokenized[idx];
wait_before_errors("Tokenization error found. Press Enter to dump tokens...");
eprintln!("--- Tokenization error in: {}", bad_path.display());
//bad_tf.dump_debug_layout(); // from DebugTools
let (bad_path, bad_tf) = &tokenized[idx];
wait_before_errors("Tokenization issues detected. Press Enter to print diagnostics...");
let fname = bad_path.display().to_string();
eprintln!("--- Tokenization issues in first failing file ---");
eprintln!("File: {fname}");
let diags = bad_tf.diagnostics();
if diags.is_empty() {
eprintln!("(no diagnostics captured)");
} else {
render_diagnostics_window(diags, bad_tf, &fname);
}
std::process::exit(1);
}
@ -298,38 +346,8 @@ fn main() {
if diags.is_empty() && fatal.is_none() {
eprintln!("(no diagnostics captured)");
} else {
let use_colors = is_terminal::is_terminal(io::stderr());
let fname = path.display().to_string();
let total = diags.len();
let first_n = DIAG_SHOW_FIRST.min(total);
let last_n = DIAG_SHOW_LAST.min(total.saturating_sub(first_n));
if total > first_n + last_n {
// first window
for (k, d) in diags.iter().take(first_n).enumerate() {
let s = pretty::render_diagnostic(d, tf, Some(&fname), use_colors);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{}: {:#?}", k + 1, d);
}
}
eprintln!("... {} diagnostics omitted ...", total - (first_n + last_n));
// last window
let start = total - last_n;
for (offset, d) in diags.iter().skip(start).enumerate() {
let idx_global = start + offset + 1;
let s = pretty::render_diagnostic(d, tf, Some(&fname), use_colors);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{idx_global}: {d:#?}");
}
}
} else {
for (k, d) in diags.iter().enumerate() {
let s = pretty::render_diagnostic(d, tf, Some(&fname), use_colors);
if ALSO_PRINT_DEBUG_AFTER_PRETTY {
eprintln!("#{}: {:#?}", k + 1, d);
}
}
}
render_diagnostics_window(&diags, tf, &fname);
}
std::process::exit(1);
}

136
dev_tests/src/verify_expr.rs
View file

@ -7,69 +7,63 @@
)]
use rottlib::arena::Arena;
use rottlib::diagnostics::Diagnostic;
use rottlib::lexer::TokenizedFile;
use rottlib::parser::Parser;
mod pretty;
// a * * *
/// Expressions to test.
/// Lexer-focused fixtures.
///
/// Add, remove, or edit entries here.
/// Using `(&str, &str)` gives each case a human-readable label.
/// Expressions to test.
///
/// Add, remove, or edit entries here.
/// Using `(&str, &str)` gives each case a human-readable label.
/// Keep these small: the goal is to inspect lexer diagnostics and delimiter
/// recovery behavior, not full parser behavior.
const TEST_CASES: &[(&str, &str)] = &[
// P0022: invalid return value start
// L0001: invalid or unknown token
(
"files/P0022_01.uc",
"return ] ;",
),
(
"files/P0022_02.uc",
"return\n ]\n;\n",
),
(
"files/P0022_03.uc",
"return\n}\n",
"files/L0001_01.uc",
"`",
),
// P0023: invalid break value start
// L0002: unexpected closing delimiter
(
"files/P0023_01.uc",
"break ] ;",
),
(
"files/P0023_02.uc",
"break\n \n\n\n\n ]\n;\n",
),
(
"files/P0023_03.uc",
"break\n}\n",
"files/L0002_01.uc",
"]",
),
// P0024: goto target is missing or not a label token
// L0003: unclosed delimiter before later closing delimiter
//
// The `}` can still recover by matching the earlier `{`.
(
"files/P0024_01.uc",
"goto;",
"files/L0003_01.uc",
"{\n foo(\n}\n",
),
// L0004: mismatched closing delimiter
(
"files/P0024_02.uc",
"goto\n ;\n",
"files/L0004_01.uc",
"(]",
),
// L0005: unclosed delimiter at end of file
(
"files/P0024_03.uc",
"goto\n ]\n;\n",
"files/L0005_01.uc",
"foo(",
),
// Mixed recovery case:
//
// `)` recovers by matching `(` after treating `[` as unclosed;
// the following `]` is then unexpected.
(
"files/P0024_04.uc",
"goto",
"files/L_mixed_01.uc",
"([)]",
),
];
/// If true, also run the parser after tokenization.
///
/// For lexer-focused fixtures this is usually noisy, so keep it off unless you
/// want to inspect how parser recovery behaves after lexer diagnostics.
const RUN_PARSER: bool = false;
/// If true, print the parsed expression using Debug formatting.
const PRINT_PARSED_EXPR: bool = false;
@ -82,24 +76,46 @@ fn main() {
let mut had_any_problem = false;
for (idx, (label, source)) in TEST_CASES.iter().enumerate() {
println!("============================================================");
println!("Case {}: {}", idx + 1, label);
println!("------------------------------------------------------------");
let tf = TokenizedFile::tokenize(source);
let mut parser = Parser::new(&tf, &arena);
let expr = parser.parse_expression();
let lexer_diagnostics = tf.diagnostics();
if PRINT_PARSED_EXPR {
println!("Parsed expression:");
println!("{expr:#?}");
}
if parser.diagnostics.is_empty() {
println!("Diagnostics: none");
if lexer_diagnostics.is_empty() {
println!("Lexer diagnostics: none");
} else {
had_any_problem = true;
if ALWAYS_PRINT_DIAGNOSTICS {
let use_colors = false;
for (k, diag) in parser.diagnostics.iter().enumerate() {
pretty::render_diagnostic(diag, &tf, Some(label), use_colors);
println!("Lexer diagnostics:");
for diag in lexer_diagnostics {
render_diagnostic(diag, &tf, Some(label), false);
}
}
}
if RUN_PARSER {
let mut parser = Parser::new(&tf, &arena);
let expr = parser.parse_expression();
if PRINT_PARSED_EXPR {
println!("Parsed expression:");
println!("{expr:#?}");
}
if parser.diagnostics.is_empty() {
println!("Parser diagnostics: none");
} else {
had_any_problem = true;
if ALWAYS_PRINT_DIAGNOSTICS {
println!("Parser diagnostics:");
for diag in &parser.diagnostics {
render_diagnostic(diag, &tf, Some(label), false);
}
}
}
}
@ -108,10 +124,20 @@ fn main() {
}
println!("============================================================");
if had_any_problem {
println!("Done. At least one case had tokenization or parse diagnostics.");
println!("Done. At least one case had lexer or parser diagnostics.");
std::process::exit(1);
} else {
println!("Done. All cases completed without diagnostics.");
}
}
fn render_diagnostic(
diag: &Diagnostic,
file: &TokenizedFile<'_>,
file_name: Option<&str>,
_colors: bool,
) {
diag.render(file, file_name.unwrap_or("<default>"));
}