diff --git a/testing/Cargo.toml b/testing/Cargo.toml
index 0fcee8c0d..d676bef02 100644
--- a/testing/Cargo.toml
+++ b/testing/Cargo.toml
@@ -16,6 +16,7 @@ serde_json = { version = "1.0", optional = true }
 [dev-dependencies]
 criterion = "0.3"
 trybuild = "1.0"
+version_check = "0.9"
 
 [[bench]]
 name = "all"
diff --git a/testing/tests/ui.rs b/testing/tests/ui.rs
index 17c22482e..39a0b78ad 100644
--- a/testing/tests/ui.rs
+++ b/testing/tests/ui.rs
@@ -1,7 +1,14 @@
 use trybuild::TestCases;
+use version_check as rustc;
 
 #[cfg_attr(not(windows), test)]
 fn ui() {
     let t = TestCases::new();
     t.compile_fail("tests/ui/*.rs");
+
+    if rustc::is_min_version("1.54").unwrap() {
+        t.compile_fail("tests/ui/since_1_54/*.rs");
+    } else {
+        t.compile_fail("tests/ui/before_1_54/*.rs");
+    }
 }
diff --git a/testing/tests/ui/incorrect_path.rs b/testing/tests/ui/before_1_54/incorrect_path.rs
similarity index 100%
rename from testing/tests/ui/incorrect_path.rs
rename to testing/tests/ui/before_1_54/incorrect_path.rs
diff --git a/testing/tests/ui/incorrect_path.stderr b/testing/tests/ui/before_1_54/incorrect_path.stderr
similarity index 100%
rename from testing/tests/ui/incorrect_path.stderr
rename to testing/tests/ui/before_1_54/incorrect_path.stderr
diff --git a/testing/tests/ui/since_1_54/incorrect_path.rs b/testing/tests/ui/since_1_54/incorrect_path.rs
new file mode 120000
index 000000000..224f517b4
--- /dev/null
+++ b/testing/tests/ui/since_1_54/incorrect_path.rs
@@ -0,0 +1 @@
+../before_1_54/incorrect_path.rs
\ No newline at end of file
diff --git a/testing/tests/ui/since_1_54/incorrect_path.stderr b/testing/tests/ui/since_1_54/incorrect_path.stderr
new file mode 100644
index 000000000..e3d7ca4e3
--- /dev/null
+++ b/testing/tests/ui/since_1_54/incorrect_path.stderr
@@ -0,0 +1,7 @@
+error: template "thisdoesnotexist.html" not found in directories ["$WORKSPACE/target/tests/askama_testing/templates"]
+ --> $DIR/incorrect_path.rs:3:10
+  |
+3 | #[derive(Template)]
+  |          ^^^^^^^^
+  |
+  = note: this error originates in the derive macro `Template` (in Nightly builds, run with -Z macro-backtrace for more info)