diff --git a/Dockerfile b/Dockerfile
index f5c563fd785..41dd1d8cdfc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,6 +46,9 @@ RUN apt-get update -y && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
+# To keep mooncake can be found
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 RUN pip config set global.index-url ${PIP_INDEX_URL}
 
 # Install vLLM
diff --git a/Dockerfile.a3 b/Dockerfile.a3
index e9bf3a24576..1cbc6569fdd 100644
--- a/Dockerfile.a3
+++ b/Dockerfile.a3
@@ -47,6 +47,9 @@ RUN apt-get update -y && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
+# To keep mooncake can be found
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
 ARG VLLM_TAG=v0.16.0
diff --git a/Dockerfile.a3.openEuler b/Dockerfile.a3.openEuler
index 39509407d32..e83bc5c9bf6 100644
--- a/Dockerfile.a3.openEuler
+++ b/Dockerfile.a3.openEuler
@@ -48,6 +48,9 @@ RUN yum update -y && \
     rm -fr /vllm-workspace/Mooncake/build && \
     rm -rf /var/cache/yum/*
 
+# To keep mooncake can be found
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
 ARG VLLM_TAG=v0.16.0
diff --git a/Dockerfile.openEuler b/Dockerfile.openEuler
index 35437865bdc..50c776166d8 100644
--- a/Dockerfile.openEuler
+++ b/Dockerfile.openEuler
@@ -48,6 +48,9 @@ RUN yum update -y && \
     rm -fr /vllm-workspace/Mooncake/build && \
     rm -rf /var/cache/yum/*
 
+# To keep mooncake can be found
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
 ARG VLLM_TAG=v0.16.0