[Bug] issue while compiling FP8 dataype of mlc-ai/Llama-3.1-8B-Instruct-fp8-MLC

[Vinaysukhesh98]

Compilation complete! Exporting to disk
Traceback (most recent call last):
File "", line 198, in run_module_as_main
File "", line 88, in run_code
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/main.py", line 64, in
main()
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/main.py", line 33, in main
cli.main(sys.argv[2:])
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/cli/compile.py", line 129, in main
compile(
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/compile.py", line 243, in compile
compile(args, model_config)
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/compile.py", line 188, in compile
args.build_func(
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/support/auto_target.py", line 190, in build
ex = relax.build(
^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/python/tvm/relax/vm_build.py", line 353, in build
return vmlink(
^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/python/tvm/relax/vm_build.py", line 249, in vmlink
lib = tvm.build(
^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/python/tvm/driver/build_module.py", line 297, in build
rt_mod_host = driver_ffi.tir_to_runtime(annotated_mods, target_host)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/python/tvm/ffi/ctypes/packed_func.py", line 245, in call
raise_last_ffi_error()
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/python/tvm/ffi/base.py", line 481, in raise_last_ffi_error
raise py_err
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc", line 531, in operator()
return TIRToRuntime(inputs_arg, host_target);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc", line 492, in tvm::TIRToRuntime(tvm::runtime::Map<tvm::Target, tvm::IRModule, void, void> const&, tvm::Target const&)
auto pair = SplitMixedModule(ir_module, target, target_host);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc", line 418, in tvm::SplitMixedModule(tvm::IRModule, tvm::Target const&, tvm::Target const&)
mod_mixed = ApplyPasses(mod_mixed, MixedModulePassManager(mod_mixed, target));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc", line 291, in tvm::ApplyPasses(tvm::IRModule, tvm::transform::Sequential)
mod = seq(std::move(mod));
^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/transform.cc", line 101, in tvm::tir::transform::PrimFuncPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
func = pass_func(std::move(func), mod, pass_ctx);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 741, in operator()
return FP8ComputeLegalizer(DataType(String2DLDataType(promote_dtype_str))).Legalize(f);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 468, in tvm::tir::FP8ComputeLegalizer::Legalize(tvm::tir::PrimFunc)
return LegalizeWithPlanner(func, &planner);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 173, in tvm::tir::ComputeLegalizer::LegalizeWithPlanner(tvm::tir::PrimFunc, tvm::tir::ComputeLegalizePlanner*)
n->body = this->VisitStmt(std::move(n->body));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 340, in tvm::tir::ComputeLegalizer::VisitStmt(tvm::tir::AttrStmtNode const*)
Stmt ret = StmtExprMutator::VisitStmt(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 382, in tvm::tir::ComputeLegalizer::VisitStmt(tvm::tir::AllocateNode const*)
Stmt ret = StmtExprMutator::VisitStmt(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 382, in tvm::tir::ComputeLegalizer::VisitStmt(tvm::tir::AllocateNode const*)
Stmt ret = StmtExprMutator::VisitStmt(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 382, in tvm::tir::ComputeLegalizer::VisitStmt(tvm::tir::AllocateNode const*)
Stmt ret = StmtExprMutator::VisitStmt(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 382, in tvm::tir::ComputeLegalizer::VisitStmt(tvm::tir::AllocateNode const*)
Stmt ret = StmtExprMutator::VisitStmt(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 340, in tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
Stmt ret = StmtExprMutator::VisitStmt_(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 340, in tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
Stmt ret = StmtExprMutator::VisitStmt_(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 340, in tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
Stmt ret = StmtExprMutator::VisitStmt_(op);
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 211, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
return MutateArray(self, arr, fmutate);
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 184, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
Array copy = arr.Map(fmutate);
^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 652, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
return Array(MapHelper(data_, fmap));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 823, in tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
U mapped = fmap(DowncastNoCheck(it));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 210, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
auto fmutate = [self](const Stmt& s) { return self->VisitStmt(s); };
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 211, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
return MutateArray(self, arr, fmutate);
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 184, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
Array copy = arr.Map(fmutate);
^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 652, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
return Array(MapHelper(data_, fmap));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 823, in tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
U mapped = fmap(DowncastNoCheck(it));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 210, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
auto fmutate = [self](const Stmt& s) { return self->VisitStmt(s); };
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 301, in tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::LetStmtNode const*)
Stmt body = VisitStmt(op->body);
^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 211, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
return MutateArray(self, arr, fmutate);
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 184, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
Array copy = arr.Map(fmutate);
^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 652, in tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
return Array(MapHelper(data_, fmap));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h", line 823, in tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
U mapped = fmap(DowncastNoCheck(it));
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc", line 210, in tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
auto fmutate = [self](const Stmt& s) { return self->VisitStmt(s); };
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 330, in tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::BufferStoreNode const*)
ICHECK(MatchDType(value->dtype));
^^^^^^^^^^^^^^^^^^^^^^^^
tvm.error.InternalError: Traceback (most recent call last):
32: operator()
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc:531
31: tvm::TIRToRuntime(tvm::runtime::Map<tvm::Target, tvm::IRModule, void, void> const&, tvm::Target const&)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc:492
30: tvm::SplitMixedModule(tvm::IRModule, tvm::Target const&, tvm::Target const&)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc:418
29: tvm::ApplyPasses(tvm::IRModule, tvm::transform::Sequential)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/driver/driver_api.cc:291
28: tvm::tir::transform::PrimFuncPassNode::operator()(tvm::IRModule, tvm::transform::PassContext const&) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/transform.cc:101
27: operator()
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:741
26: tvm::tir::FP8ComputeLegalizer::Legalize(tvm::tir::PrimFunc)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:468
25: tvm::tir::ComputeLegalizer::LegalizeWithPlanner(tvm::tir::PrimFunc, tvm::tir::ComputeLegalizePlanner*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:173
24: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:340
23: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AllocateNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:382
22: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AllocateNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:382
21: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AllocateNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:382
20: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AllocateNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:382
19: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:340
18: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:340
17: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::AttrStmtNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:340
16: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:211
15: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:184
14: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:652
13: tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:823
12: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:210
11: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:211
10: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:184
9: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:652
8: tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:823
7: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:210
6: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::LetStmtNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:301
5: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:211
4: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::tir::StmtMutator::Internal::MutateArray<tvm::tir::Stmt, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}>(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> const&, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:184
3: tvm::runtime::Array<tvm::tir::Stmt, std::enable_if<std::is_base_of<tvm::runtime::ObjectRef, tvm::tir::Stmt>::value, void>::type> tvm::runtime::Array<tvm::tir::Stmt, void>::Map<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:652
2: tvm::runtime::ObjectPtrtvm::runtime::Object tvm::runtime::Array<tvm::tir::Stmt, void>::MapHelper<tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}, tvm::tir::Stmt>(tvm::runtime::ObjectPtrtvm::runtime::Object, tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1})
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/include/tvm/runtime/container/array.h:823
1: tvm::tir::StmtMutator::Internal::Mutate(tvm::tir::StmtMutator*, tvm::runtime::Array<tvm::tir::Stmt, void> const&)::{lambda(tvm::tir::Stmt const&)#1}::operator()(tvm::tir::Stmt const&) const
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/ir/stmt_functor.cc:210
0: tvm::tir::ComputeLegalizer::VisitStmt_(tvm::tir::BufferStoreNode const*)
at / Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc:330
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/3rdparty/tvm/src/tir/transforms/unsupported_dtype_legalize.cc", line 330
InternalError: Check failed: (MatchDType(value->dtype)) is false:
Traceback (most recent call last):
File "", line 198, in _run_module_as_main
File "", line 88, in _run_code
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/main.py", line 64, in
main()
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/main.py", line 53, in main
cli.main(sys.argv[2:])
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/cli/package.py", line 64, in main
package(
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/package.py", line 351, in package
model_lib_path_for_prepare_libs = build_model_library(
^^^^^^^^^^^^^^^^^^^^
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/package.py", line 92, in build_model_library
jit.jit(
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/jit.py", line 164, in jit
_run_jit(
File "/ Documents/mlc-chat_TVM_Oct24/mlc-llm/python/mlc_llm/interface/jit.py", line 124, in _run_jit
raise RuntimeError("Cannot find compilation output, compilation failed")
RuntimeError: Cannot find compilation output, compilation failed

[MasterJH5574]

Hi @Vinaysukhesh98, may I ask which what's the GPU device on your side? Right now the FP8 support is only available on NVIDIA H100 series that have compute capability at least compute_90. From the backtrace you share, it looks like the hardware does not support FP8. If that's the case then we are not able to run models with FP8 precision on the hardware.

[Vinaysukhesh98]

Thanks @MasterJH5574
Okay i am using the vulkan and opencl gpu libs that why i am getting the above error, even if we compile this the model dosent run on any mobile devices right

[MasterJH5574]

Yeah that's right, compiling requires CUDA support. If you want to specify the compilation to GPUs with compute capability at least compute_90, you may use the knob --device nvidia/nvidia-h100 in the mlc_llm compile command.