diff --git a/src/auditwheel/lddtree.py b/src/auditwheel/lddtree.py
index 2c03259b..ffd31a1a 100644
--- a/src/auditwheel/lddtree.py
+++ b/src/auditwheel/lddtree.py
@@ -386,6 +386,7 @@ def lddtree(
         libs: list[str] = []
         rpaths: list[str] = []
         runpaths: list[str] = []
+        _excluded_libs: set[str] = set()
         for segment in elf.iter_segments():
             if segment.header.p_type != "PT_DYNAMIC":
                 continue
@@ -396,8 +397,11 @@ def lddtree(
                 elif t.entry.d_tag == "DT_RUNPATH":
                     runpaths = parse_ld_paths(t.runpath, path=path, root=root)
                 elif t.entry.d_tag == "DT_NEEDED":
-                    if any(fnmatch(t.needed, e) for e in exclude):
+                    if t.needed in _excluded_libs or any(
+                        fnmatch(t.needed, e) for e in exclude
+                    ):
                         log.info("Excluding %s", t.needed)
+                        _excluded_libs.add(t.needed)
                     else:
                         libs.append(t.needed)
             if runpaths:
@@ -416,7 +420,6 @@ def lddtree(
             log.debug("  ldpaths[runpath] = %s", runpaths)
         ret["rpath"] = rpaths
         ret["runpath"] = runpaths
-        ret["needed"] = libs
 
         # Search for the libs this ELF uses.
         all_ldpaths: list[str] | None = None
@@ -434,6 +437,12 @@ def lddtree(
                     + ldpaths["interp"]
                 )
             realpath, fullpath = find_lib(elf, lib, all_ldpaths, root)
+            if lib in _excluded_libs or (
+                realpath is not None and any(fnmatch(realpath, e) for e in exclude)
+            ):
+                log.info("Excluding %s", realpath)
+                _excluded_libs.add(lib)
+                continue
             _all_libs[lib] = {
                 "realpath": realpath,
                 "path": fullpath,
@@ -453,4 +462,6 @@ def lddtree(
 
         del elf
 
+        ret["needed"] = [lib for lib in libs if lib not in _excluded_libs]
+
     return ret
diff --git a/tests/integration/libffi.so.5 b/tests/integration/libffi.so.5
new file mode 100755
index 00000000..190e7af1
Binary files /dev/null and b/tests/integration/libffi.so.5 differ
diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py
index 13adb7bd..b4d14a0f 100644
--- a/tests/integration/test_bundled_wheels.py
+++ b/tests/integration/test_bundled_wheels.py
@@ -1,18 +1,21 @@
 from __future__ import annotations
 
+import importlib
+import os
 import platform
 import subprocess
 import sys
 import zipfile
 from argparse import Namespace
 from datetime import datetime, timezone
+from os.path import isabs
 from pathlib import Path
 from unittest import mock
 from unittest.mock import Mock
 
 import pytest
 
-from auditwheel import main_repair
+from auditwheel import lddtree, main_repair
 from auditwheel.libc import Libc
 from auditwheel.policy import WheelPolicies
 from auditwheel.wheel_abi import analyze_wheel_abi
@@ -40,6 +43,11 @@
             {"libffi.so.5"},
             frozenset(["libffi.so.[6,7]"]),
         ),
+        (
+            "cffi-1.5.0-cp27-none-linux_x86_64.whl",
+            set(),
+            frozenset([f"{HERE}/*"]),
+        ),
         ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["libffi.so.*"])),
         ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["*"])),
         (
@@ -50,9 +58,23 @@
     ],
 )
 def test_analyze_wheel_abi(file, external_libs, exclude):
-    wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch="x86_64")
-    winfo = analyze_wheel_abi(wheel_policies, str(HERE / file), exclude)
-    assert set(winfo.external_refs["manylinux_2_5_x86_64"]["libs"]) == external_libs
+    # If exclude libs contain path, LD_LIBRARY_PATH need to be modified to find the libs
+    # `lddtree.load_ld_paths` needs to be reloaded for it's `lru_cache`-ed.
+    modify_ld_library_path = any(isabs(e) for e in exclude)
+
+    with pytest.MonkeyPatch.context() as cp:
+        if modify_ld_library_path:
+            cp.setenv("LD_LIBRARY_PATH", f"{HERE}")
+            importlib.reload(lddtree)
+
+        wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch="x86_64")
+        winfo = analyze_wheel_abi(wheel_policies, str(HERE / file), exclude)
+        assert (
+            set(winfo.external_refs["manylinux_2_5_x86_64"]["libs"]) == external_libs
+        ), f"{HERE}, {exclude}, {os.environ}"
+
+    if modify_ld_library_path:
+        importlib.reload(lddtree)
 
 
 def test_analyze_wheel_abi_pyfpe():