diff --git a/controllers/openstackfloatingippool_controller.go b/controllers/openstackfloatingippool_controller.go
index 6ee65140a3..7e2e78f612 100644
--- a/controllers/openstackfloatingippool_controller.go
+++ b/controllers/openstackfloatingippool_controller.go
@@ -456,6 +456,9 @@ func (r *OpenStackFloatingIPPoolReconciler) SetupWithManager(ctx context.Context
 
 	if err := mgr.GetFieldIndexer().IndexField(ctx, &ipamv1.IPAddress{}, infrav1alpha1.OpenStackFloatingIPPoolNameIndex, func(rawObj client.Object) []string {
 		ip := rawObj.(*ipamv1.IPAddress)
+		if ip.Spec.PoolRef.Kind != openStackFloatingIPPool {
+			return nil
+		}
 		return []string{ip.Spec.PoolRef.Name}
 	}); err != nil {
 		return err