diff --git a/cmake/vcpkg-ports/pybind11/portfile.cmake b/cmake/vcpkg-ports/pybind11/portfile.cmake
index 4e4cd30a26df1..e0dd402cf186b 100644
--- a/cmake/vcpkg-ports/pybind11/portfile.cmake
+++ b/cmake/vcpkg-ports/pybind11/portfile.cmake
@@ -1,10 +1,15 @@
-vcpkg_from_github(
-    OUT_SOURCE_PATH SOURCE_PATH
-    REPO pybind/pybind11
-    REF "v${VERSION}"
-    # SHA512 for the zip (not tar.gz) file.
+# Manually define the download for the .zip archive (to be consistent with deps.txt)
+# If we used vcpkg_from_github, it would download the .tar.gz archive,
+# which has different SHA512: 19bee2c76320e25202ee078b5680ff8a7acfb33494dec29dad984ab04de8bcb01340d9fec37c8cc5ac9015dfc367e60312dcd8506e66ce8f0af4c49db562ddef
+vcpkg_download_distfile(ARCHIVE
+    URLS "https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.zip"
+    FILENAME "pybind11-${VERSION}.zip"
     SHA512 786b1bf534ac67a8d5669f8babf67bb13e48b3a3da1b6344e43ae10a84b80bbc8fea5f12a65fd18739c341fefef5622c5dc096db964dff33cc62ea4259b2e2c1
-    HEAD_REF master
+)
+
+vcpkg_extract_source_archive(
+    SOURCE_PATH
+    ARCHIVE "${ARCHIVE}"
 )
 
 vcpkg_cmake_configure(