diff --git a/cranelift/isle/isle/src/sema.rs b/cranelift/isle/isle/src/sema.rs
index 9a6641fb61..b3d0a6f543 100644
--- a/cranelift/isle/isle/src/sema.rs
+++ b/cranelift/isle/isle/src/sema.rs
@@ -1034,6 +1034,16 @@ impl TermEnv {
                         return None;
                     }
                 };
+                if !tyenv.types[ty.index()].is_prim() {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "expected non-primitive type {}, but found integer literal '{}'",
+                            tyenv.types[ty.index()].name(tyenv),
+                            val,
+                        ),
+                    );
+                }
                 Some((Pattern::ConstInt(ty, val), ty))
             }
             &ast::Pattern::ConstPrim { ref val, pos } => {