Initial commit

dev_tests/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "dev_tests"
+version = "0.1.0"
+edition = "2024"
+
+[[bin]]
+name = "dump_tokens"
+path = "src/dump_tokens.rs"
+
+[[bin]]
+name = "uc_lexer_verify"
+path = "src/uc_lexer_verify.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+rottlib = { version = "0", path = "../rottlib", features = ["debug"] }
+walkdir="2.5"
+encoding_rs="0.8"
+chardet="0.2"
+
+[lints]
+workspace = true

dev_tests/src/dump_tokens.rs

@@ -0,0 +1,76 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+
+use encoding_rs::{Encoding, UTF_8};
+use rottlib::lexer::{DebugTools, TokenizedFile};
+
+/// Recursively search `root` for the first file whose *basename* matches
+/// `needle` (case-sensitive).
+///
+/// Returns the absolute path.
+fn find_file(root: &Path, needle: &str) -> Option<PathBuf> {
+    for entry in walkdir::WalkDir::new(root)
+        .into_iter()
+        .filter_map(Result::ok)
+    {
+        let path = entry.path();
+        if path.is_file() && (path.file_name().and_then(|name| name.to_str()) == Some(needle)) {
+            return fs::canonicalize(path).ok();
+        }
+    }
+    None
+}
+
+/// CLI: `dump_tokens <root_dir> <file_name>` - searches for `<file_name>`
+/// recursively inside `<root_dir>`.
+///
+/// This utility takes *root directory* and *file name* instead of the full path
+/// to help us avoid searching for them typing names out:
+///
+/// - We know where all the sources are;
+/// - We usually just know the name of the file that is being problematic.
+fn main() {
+    let mut args = std::env::args().skip(1);
+    let root_dir = args.next().unwrap_or_else(|| {
+        eprintln!("Usage: inspect_uc <root_dir> <file_name>");
+        std::process::exit(1);
+    });
+    let file_name = args.next().unwrap_or_else(|| {
+        eprintln!("Usage: inspect_uc <root_dir> <file_name>");
+        std::process::exit(1);
+    });
+
+    let root = PathBuf::from(&root_dir);
+    if !root.exists() {
+        eprintln!("Root directory '{root_dir}' does not exist.");
+        std::process::exit(1);
+    }
+
+    let found_path = find_file(&root, &file_name).map_or_else(
+        || {
+            eprintln!("File '{file_name}' not found under '{root_dir}'.");
+            std::process::exit(1);
+        },
+        |path| path,
+    );
+
+    // Read & decode
+    let raw_bytes = match fs::read(&found_path) {
+        Ok(sources) => sources,
+        Err(error) => {
+            eprintln!("Could not read {}: {error}", found_path.display());
+            std::process::exit(1);
+        }
+    };
+
+    let (encoding_label, _, _) = chardet::detect(&raw_bytes);
+    let encoding = Encoding::for_label(encoding_label.as_bytes()).unwrap_or(UTF_8);
+    let (decoded_str, _, _) = encoding.decode(&raw_bytes);
+
+    let source_text = decoded_str.to_string();
+    let tokenized_file = TokenizedFile::from_source(&source_text);
+
+    tokenized_file.dump_debug_layout();
+}

dev_tests/src/uc_lexer_verify.rs

@@ -0,0 +1,122 @@
+use std::{collections::HashSet, fs, path::PathBuf};
+
+use rottlib::lexer::{DebugTools, TokenizedFile};
+
+/// Read `ignore.txt` (one path per line, `#` for comments) from root directory
+/// and turn it into a canonicalized [`HashSet<PathBuf>`].
+fn load_ignore_set(root: &std::path::Path) -> HashSet<PathBuf> {
+    let ignore_file = root.join("ignore.txt");
+    if !ignore_file.exists() {
+        return HashSet::new();
+    }
+
+    let content = match fs::read_to_string(&ignore_file) {
+        Ok(content) => content,
+        Err(error) => {
+            eprintln!("Could not read {}: {error}", ignore_file.display());
+            return HashSet::new();
+        }
+    };
+
+    content
+        .lines()
+        .map(str::trim)
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .filter_map(|line| {
+            let next_path = PathBuf::from(line);
+            let absolute_path = if next_path.is_absolute() {
+                next_path
+            } else {
+                root.join(next_path)
+            };
+            fs::canonicalize(absolute_path).ok()
+        })
+        .collect()
+}
+
+/// CLI: `verify_uc <root_dir>` - find all `.uc` files in the provided directory
+/// (except those listed in `ignore.txt` in the root) and test them all.
+///
+/// Reported execution time is the tokenization time, without considering time
+/// it takes to read files from disk.
+///
+/// `ignore.txt` is for listing specific files, not directories.
+fn main() {
+    let root_dir = std::env::args().nth(1).unwrap(); // it is fine to crash debug utility
+    let root = PathBuf::from(&root_dir);
+
+    if !root.exists() {
+        eprintln!("Root directory '{root_dir}' does not exist.");
+        std::process::exit(1);
+    }
+
+    // Load files
+    let ignored_paths = load_ignore_set(&root);
+    let mut uc_files: Vec<(PathBuf, String)> = Vec::new();
+    for entry in walkdir::WalkDir::new(&root)
+        .into_iter()
+        .filter_map(Result::ok) // for debug tool this is ok
+        .filter(|entry| {
+            let path = entry.path();
+            // Skip anything explicitly ignored
+            if let Ok(absolute_path) = fs::canonicalize(path) {
+                if ignored_paths.contains(&absolute_path) {
+                    return false;
+                }
+            }
+            // Must be *.uc
+            path.is_file()
+                && path
+                    .extension()
+                    .and_then(|extension| extension.to_str())
+                    .is_some_and(|extension| extension.eq_ignore_ascii_case("uc"))
+        })
+    {
+        let path = entry.path();
+        match fs::read(path) {
+            Ok(raw_bytes) => {
+                // Auto‑detect encoding for old Unreal script sources
+                let (encoding_label, _, _) = chardet::detect(&raw_bytes);
+                let encoding = encoding_rs::Encoding::for_label(encoding_label.as_bytes())
+                    .unwrap_or(encoding_rs::UTF_8);
+                let (decoded_text, _, _) = encoding.decode(&raw_bytes);
+                uc_files.push((path.to_path_buf(), decoded_text.into_owned()));
+            }
+            Err(error) => {
+                eprintln!("Failed to read `{}`: {error}", path.display());
+                std::process::exit(1);
+            }
+        }
+    }
+    println!("Loaded {} .uc files into memory.", uc_files.len());
+
+    // Tokenize and measure performance
+    let start_time = std::time::Instant::now();
+    let tokenized_files: Vec<(PathBuf, TokenizedFile)> = uc_files
+        .iter()
+        .map(|(path, source_code)| {
+            let tokenized_file = TokenizedFile::from_source(source_code);
+            if tokenized_file.had_errors() {
+                println!("TK: {}", path.display());
+            }
+            (path.clone(), tokenized_file)
+        })
+        .collect();
+    let elapsed_time = start_time.elapsed();
+    println!(
+        "Tokenized {} files in {:.2?}",
+        tokenized_files.len(),
+        elapsed_time
+    );
+
+    // Round‑trip check
+    for ((path, original), (_, tokenized_file)) in uc_files.iter().zip(tokenized_files.iter()) {
+        let reconstructed = tokenized_file.reconstruct_source();
+        if original != &reconstructed {
+            eprintln!("Reconstruction mismatch in `{}`!", path.display());
+            std::process::exit(1);
+        }
+    }
+
+    println!("All .uc files matched successfully.");
+}