Add delimeter matching to lexer

rottlib/tests/lexer_diagnostics.rs

@@ -0,0 +1,350 @@
+use std::collections::HashMap;
+
+use rottlib::diagnostics::{Diagnostic, Severity};
+use rottlib::lexer::{TokenPosition, TokenSpan, TokenizedFile};
+
+#[derive(Debug)]
+struct ExpectedLabel {
+    span: TokenSpan,
+    message: &'static str,
+}
+
+#[derive(Debug)]
+struct ExpectedDiagnostic<'a> {
+    headline: &'static str,
+    severity: Severity,
+    code: Option<&'static str>,
+    primary_label: Option<ExpectedLabel>,
+    secondary_labels: &'a [ExpectedLabel],
+    help: Option<&'static str>,
+    notes: &'a [&'static str],
+}
+
+#[track_caller]
+fn assert_diagnostic(actual: &Diagnostic, expected: &ExpectedDiagnostic<'_>) {
+    assert_eq!(actual.headline(), expected.headline);
+    assert_eq!(actual.severity(), expected.severity);
+    assert_eq!(actual.code(), expected.code);
+    assert_eq!(actual.help(), expected.help);
+
+    match (actual.primary_label(), expected.primary_label.as_ref()) {
+        (None, None) => {}
+        (Some(actual), Some(expected)) => {
+            assert_eq!(actual.span, expected.span);
+            assert_eq!(actual.message, expected.message);
+        }
+        _ => panic!("primary label mismatch"),
+    }
+
+    let actual_secondary = actual.secondary_labels();
+    assert_eq!(actual_secondary.len(), expected.secondary_labels.len());
+
+    for (actual, expected) in actual_secondary
+        .iter()
+        .zip(expected.secondary_labels.iter())
+    {
+        assert_eq!(actual.span, expected.span);
+        assert_eq!(actual.message, expected.message);
+    }
+
+    let actual_notes = actual.notes();
+    assert_eq!(actual_notes.len(), expected.notes.len());
+
+    for (actual, expected) in actual_notes.iter().zip(expected.notes.iter()) {
+        assert_eq!(actual, expected);
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct Fixture {
+    label: &'static str,
+    source: &'static str,
+}
+
+type FixtureRun = Vec<Diagnostic>;
+
+struct FixtureRuns {
+    runs: HashMap<&'static str, FixtureRun>,
+}
+
+impl FixtureRuns {
+    #[track_caller]
+    fn get(&self, label: &str) -> Option<Vec<Diagnostic>> {
+        self.runs.get(label).cloned()
+    }
+
+    #[track_caller]
+    fn get_any(&self, label: &str) -> Diagnostic {
+        self.runs
+            .get(label)
+            .map(|fixture_run| fixture_run[0].clone())
+            .unwrap_or_else(|| panic!("no fixture run for `{label}`"))
+    }
+
+    #[track_caller]
+    fn get_by_code(&self, label: &str, code: &str) -> Diagnostic {
+        self.runs
+            .get(label)
+            .unwrap_or_else(|| panic!("no fixture run for `{label}`"))
+            .iter()
+            .find(|diagnostic| diagnostic.code() == Some(code))
+            .unwrap_or_else(|| panic!("no `{code}` diagnostic in fixture `{label}`"))
+            .clone()
+    }
+}
+
+const fn span(position: usize) -> TokenSpan {
+    TokenSpan {
+        start: TokenPosition(position),
+        end: TokenPosition(position),
+    }
+}
+
+const LEXER_FIXTURES: &[Fixture] = &[
+    Fixture {
+        label: "files/L0001_01.uc",
+        source: "`",
+    },
+    Fixture {
+        label: "files/L0002_01.uc",
+        source: "]",
+    },
+    Fixture {
+        label: "files/L0003_01.uc",
+        source: "{\n    foo(\n}\n",
+    },
+    Fixture {
+        label: "files/L0004_01.uc",
+        source: "(]",
+    },
+    Fixture {
+        label: "files/L0005_01.uc",
+        source: "foo(",
+    },
+    Fixture {
+        label: "files/L_mixed_01.uc",
+        source: "([)]",
+    },
+];
+
+fn run_fixture(fixture: &'static Fixture) -> FixtureRun {
+    let file = TokenizedFile::tokenize(fixture.source);
+    file.diagnostics().to_vec()
+}
+
+fn run_fixtures(fixtures: &'static [Fixture]) -> FixtureRuns {
+    let mut runs = HashMap::new();
+
+    for fixture in fixtures {
+        runs.insert(fixture.label, run_fixture(fixture));
+    }
+
+    FixtureRuns { runs }
+}
+
+#[test]
+fn check_lexer_diagnostic_counts() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_eq!(runs.get("files/L0001_01.uc").unwrap().len(), 1);
+    assert_eq!(runs.get("files/L0002_01.uc").unwrap().len(), 1);
+    assert_eq!(runs.get("files/L0003_01.uc").unwrap().len(), 1);
+    assert_eq!(runs.get("files/L0004_01.uc").unwrap().len(), 1);
+    assert_eq!(runs.get("files/L0005_01.uc").unwrap().len(), 1);
+    assert_eq!(runs.get("files/L_mixed_01.uc").unwrap().len(), 2);
+}
+
+#[test]
+fn check_l0001_invalid_token() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_any("files/L0001_01.uc"),
+        &ExpectedDiagnostic {
+            headline: "invalid token: backtick",
+            severity: Severity::Error,
+            code: Some("L0001"),
+            primary_label: Some(ExpectedLabel {
+                span: span(0),
+                message: "invalid token: backtick",
+            }),
+            secondary_labels: &[],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_l0002_unexpected_closing_delimiter() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_any("files/L0002_01.uc"),
+        &ExpectedDiagnostic {
+            headline: "unexpected closing delimiter: `]`",
+            severity: Severity::Error,
+            code: Some("L0002"),
+            primary_label: Some(ExpectedLabel {
+                span: span(0),
+                message: "unexpected closing delimiter",
+            }),
+            secondary_labels: &[],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_l0003_unclosed_delimiter_before_later_close() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_any("files/L0003_01.uc"),
+        &ExpectedDiagnostic {
+            headline: "unclosed delimiter before `}`",
+            severity: Severity::Error,
+            code: Some("L0003"),
+            primary_label: Some(ExpectedLabel {
+                span: span(4),
+                message: "this `(` is not closed before `}`",
+            }),
+            secondary_labels: &[
+                ExpectedLabel {
+                    span: span(6),
+                    message: "this `}` is matched with the earlier `{`",
+                },
+                ExpectedLabel {
+                    span: span(0),
+                    message: "this `{` is likely the intended match",
+                },
+            ],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_l0004_mismatched_closing_delimiter() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_any("files/L0004_01.uc"),
+        &ExpectedDiagnostic {
+            headline: "mismatched closing delimiter: `]`",
+            severity: Severity::Error,
+            code: Some("L0004"),
+            primary_label: Some(ExpectedLabel {
+                span: span(1),
+                message: "closing delimiter does not match `(`",
+            }),
+            secondary_labels: &[ExpectedLabel {
+                span: span(0),
+                message: "`(` opened here",
+            }],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_l0005_unclosed_delimiter_at_eof() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_any("files/L0005_01.uc"),
+        &ExpectedDiagnostic {
+            headline: "unclosed delimiter: `(`",
+            severity: Severity::Error,
+            code: Some("L0005"),
+            primary_label: Some(ExpectedLabel {
+                span: span(1),
+                message: "this `(` was never closed",
+            }),
+            secondary_labels: &[],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_mixed_recovery_diagnostics() {
+    let runs = run_fixtures(LEXER_FIXTURES);
+
+    assert_diagnostic(
+        &runs.get_by_code("files/L_mixed_01.uc", "L0003"),
+        &ExpectedDiagnostic {
+            headline: "unclosed delimiter before `)`",
+            severity: Severity::Error,
+            code: Some("L0003"),
+            primary_label: Some(ExpectedLabel {
+                span: span(1),
+                message: "this `[` is not closed before `)`",
+            }),
+            secondary_labels: &[
+                ExpectedLabel {
+                    span: span(2),
+                    message: "this `)` is matched with the earlier `(`",
+                },
+                ExpectedLabel {
+                    span: span(0),
+                    message: "this `(` is likely the intended match",
+                },
+            ],
+            help: None,
+            notes: &[],
+        },
+    );
+
+    assert_diagnostic(
+        &runs.get_by_code("files/L_mixed_01.uc", "L0002"),
+        &ExpectedDiagnostic {
+            headline: "unexpected closing delimiter: `]`",
+            severity: Severity::Error,
+            code: Some("L0002"),
+            primary_label: Some(ExpectedLabel {
+                span: span(3),
+                message: "unexpected closing delimiter",
+            }),
+            secondary_labels: &[],
+            help: None,
+            notes: &[],
+        },
+    );
+}
+
+#[test]
+fn check_recovered_delimiter_matches_are_stored() {
+    let file = TokenizedFile::tokenize("{\n    foo(\n}\n");
+
+    assert_eq!(
+        file.matching_delimiter(TokenPosition(0)),
+        Some(TokenPosition(6))
+    );
+    assert_eq!(
+        file.matching_delimiter(TokenPosition(6)),
+        Some(TokenPosition(0))
+    );
+    assert_eq!(file.matching_delimiter(TokenPosition(4)), None);
+}
+
+#[test]
+fn check_mixed_recovery_delimiter_matches_are_stored() {
+    let file = TokenizedFile::tokenize("([)]");
+
+    assert_eq!(
+        file.matching_delimiter(TokenPosition(0)),
+        Some(TokenPosition(2))
+    );
+    assert_eq!(
+        file.matching_delimiter(TokenPosition(2)),
+        Some(TokenPosition(0))
+    );
+
+    assert_eq!(file.matching_delimiter(TokenPosition(1)), None);
+    assert_eq!(file.matching_delimiter(TokenPosition(3)), None);
+}