Merge pull request #518 from shiika-lang/error-message-tests

Add tests for erroneous Shiika programs

Cargo.toml

@@ -21,16 +21,18 @@ skc_ast2hir = { path = "lib/skc_ast2hir/" }
 skc_mir = { path = "lib/skc_mir/" }
 skc_codegen = { path = "lib/skc_codegen/" }
 
-ariadne = "0.1.5"
 anyhow = "1.0"
 inkwell = { git = "https://github.com/TheDan64/inkwell", features = ["llvm16-0"], rev = "4030f76" }
 clap = { version = "3.1.18", features = ["derive"] }
-either = "1.5.3"
 env_logger = "0.8.2"
 log = "0.4.11"
 serde = { version = "1.0.125", features = ["derive"] }
 serde_json = "1.0"
 os_info = "3.7.0"
+concolor = { version = "0.1.1", features = ["api"] }
 
 chrono = "0.4"
 chrono-tz = "0.8"
+
+[dev-dependencies]
+insta = { version = "1.32.0", features = ["glob"] }

lib/shiika_parser/Cargo.toml

@@ -7,4 +7,4 @@ edition = "2021"
 shiika_core = { path = "../shiika_core" }
 shiika_ast = { path = "../shiika_ast" }
 thiserror = "1.0"
-ariadne = "0.1.5"
+ariadne = { version = "0.3.0", features = ["auto-color"] }

lib/skc_error/Cargo.toml

@@ -5,4 +5,4 @@ edition = "2021"
 
 [dependencies]
 shiika_ast = { path = "../shiika_ast" }
-ariadne = "0.1.5"
+ariadne = { version = "0.3.0", features = ["auto-color"] }

lib/skc_error/src/lib.rs

@@ -13,9 +13,9 @@ type AriadneSpan<'a> = (&'a String, Range<usize>);
 pub fn build_report<F>(main_msg: String, locs: &LocationSpan, f: F) -> String
 where
     F: for<'b> FnOnce(
-        ReportBuilder<AriadneSpan<'b>>,
+        ReportBuilder<'b, AriadneSpan<'b>>,
         AriadneSpan<'b>,
-    ) -> ReportBuilder<AriadneSpan<'b>>,
+    ) -> ReportBuilder<'b, AriadneSpan<'b>>,
 {
     if let LocationSpan::Just {
         filepath,

tests/README.md

@@ -0,0 +1,39 @@
+# Tests
+
+## `tests/sk`
+
+This directory contains various Shiika programs and run by `tests/integration_test.rs`.
+
+### Conventions
+
+.sk in this directory
+
+- must ends with `puts "ok"`
+- must not print anything other if succeed
+- should print error message if failed
+
+## `tests/erroneous`
+
+This directory contains various Shiika programs which is expected to cause compilation error.
+The expected output is stored in `snapshots` directory with the [insta crate](https://insta.rs/docs/).
+
+However this does not mean current error messages are considered perfect; PRs to improve them are welcome. 
+
+Rather than that, erroneous tests are for:
+
+- assuring the type checker detects various type errors
+- assuring the compiler does not crash with an erroneous program
+- investigating the impact of a modification to the type checker 
+
+### How to add new .sk to `tests/erroneous`
+
+prerequisites: `cargo install cargo-insta`
+
+1. Create .sk
+2. `cargo test test_erroneous` (this will create `tests/snapshots/*.snap.new`)
+3. `cargo insta review` and accept the snapshot (this will rename `.snap.new` to `.snap`)
+4. Commit `.snap` to git
+
+### How to fix CI fail after changing error message
+
+TBA

tests/erroneous.rs

@@ -0,0 +1,22 @@
+use anyhow::Result;
+use concolor;
+use insta::{assert_snapshot, glob};
+use shiika::runner;
+use std::path::Path;
+
+#[test]
+fn test_erroneous() -> Result<()> {
+    concolor::set(concolor::ColorChoice::Never);
+    let base = Path::new(".").canonicalize()?;
+    glob!("erroneous/**/*.sk", |sk_path_| {
+        // Make the path relative to the project root so that the resulting .snap will be
+        // identical on my machine and in the CI environment.
+        let sk_path = sk_path_.strip_prefix(&base).unwrap();
+        let compiler_output = match runner::compile(sk_path) {
+            Ok(_) => "".to_string(),
+            Err(comp_err) => comp_err.to_string(),
+        };
+        assert_snapshot!(compiler_output);
+    });
+    Ok(())
+}

tests/erroneous/class_definition/no_class_name.sk

@@ -0,0 +1,2 @@
+class
+end

tests/erroneous/class_definition/unknown_superclass.sk

@@ -0,0 +1,2 @@
+class A : XXX
+end

tests/erroneous/method_call/invalid_type.sk

@@ -0,0 +1 @@
+puts 123

tests/snapshots/erroneous__erroneous@class_definition__no_class_name.sk.snap

@@ -0,0 +1,13 @@
+---
+source: tests/erroneous.rs
+expression: compiler_output
+input_file: tests/erroneous/class_definition/no_class_name.sk
+---
+Error: class name must start with A-Z but got Separator
+   ╭─[tests/erroneous/class_definition/no_class_name.sk:1:6]
+   │
+ 1 │ class
+   │      ─  
+   │          
+───╯
+)

tests/snapshots/erroneous__erroneous@class_definition__unknown_superclass.sk.snap

@@ -0,0 +1,13 @@
+---
+source: tests/erroneous.rs
+expression: compiler_output
+input_file: tests/erroneous/class_definition/unknown_superclass.sk
+---
+Error: unknown type XXX in Namespace([])
+   ╭─[tests/erroneous/class_definition/unknown_superclass.sk:1:11]
+   │
+ 1 │ class A : XXX
+   │           ─┬─  
+   │            ╰─── unknown type
+───╯
+

tests/snapshots/erroneous__erroneous@method_call__invalid_type.sk.snap

@@ -0,0 +1,13 @@
+---
+source: tests/erroneous.rs
+expression: compiler_output
+input_file: tests/erroneous/method_call/invalid_type.sk
+---
+Error: the argument `str' of `Object#puts' should be String but got Int
+   ╭─[tests/erroneous/method_call/invalid_type.sk:1:6]
+   │
+ 1 │ puts 123
+   │      ─┬─  
+   │       ╰─── Int
+───╯
+