Skip to content

Implement XR_META_foveation_eye_tracked #112888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Repiteo merged 1 commit into from
Nov 18, 2025
Merged

Implement XR_META_foveation_eye_tracked #112888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 50 additions & 6 deletions drivers/vulkan/rendering_device_driver_vulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,7 @@ Error RenderingDeviceDriverVulkan::_initialize_device_extensions() {
_register_requested_device_extension(VK_KHR_MULTIVIEW_EXTENSION_NAME, false);
_register_requested_device_extension(VK_KHR_FRAGMENT_SHADING_RATE_EXTENSION_NAME, false);
_register_requested_device_extension(VK_EXT_FRAGMENT_DENSITY_MAP_EXTENSION_NAME, false);
_register_requested_device_extension(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME, false);
_register_requested_device_extension(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME, false);
_register_requested_device_extension(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME, false);
_register_requested_device_extension(VK_KHR_STORAGE_BUFFER_STORAGE_CLASS_EXTENSION_NAME, false);
Expand Down Expand Up @@ -760,6 +761,7 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
VkPhysicalDeviceVulkanMemoryModelFeaturesKHR vulkan_memory_model_features = {};
VkPhysicalDeviceFragmentShadingRateFeaturesKHR fsr_features = {};
VkPhysicalDeviceFragmentDensityMapFeaturesEXT fdm_features = {};
VkPhysicalDeviceFragmentDensityMapOffsetFeaturesQCOM fdmo_features_qcom = {};
VkPhysicalDevice16BitStorageFeaturesKHR storage_feature = {};
VkPhysicalDeviceMultiviewFeatures multiview_features = {};
VkPhysicalDevicePipelineCreationCacheControlFeatures pipeline_cache_control_features = {};
Expand Down Expand Up @@ -799,6 +801,12 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
next_features = &fdm_features;
}

if (enabled_device_extension_names.has(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME)) {
fdmo_features_qcom.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_DENSITY_MAP_OFFSET_FEATURES_QCOM;
fdmo_features_qcom.pNext = next_features;
next_features = &fdmo_features_qcom;
}

if (enabled_device_extension_names.has(VK_KHR_16BIT_STORAGE_EXTENSION_NAME)) {
storage_feature.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES_KHR;
storage_feature.pNext = next_features;
Expand Down Expand Up @@ -863,6 +871,10 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
fdm_capabilities.non_subsampled_images_supported = fdm_features.fragmentDensityMapNonSubsampledImages;
}

if (enabled_device_extension_names.has(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME)) {
fdm_capabilities.offset_supported = fdmo_features_qcom.fragmentDensityMapOffset;
}

// Multiple VRS techniques can't co-exist during the existence of one device, so we must
// choose one at creation time and only report one of them as available.
_choose_vrs_capabilities();
Expand Down Expand Up @@ -898,6 +910,7 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
void *next_properties = nullptr;
VkPhysicalDeviceFragmentShadingRatePropertiesKHR fsr_properties = {};
VkPhysicalDeviceFragmentDensityMapPropertiesEXT fdm_properties = {};
VkPhysicalDeviceFragmentDensityMapOffsetPropertiesQCOM fdmo_properties = {};
VkPhysicalDeviceMultiviewProperties multiview_properties = {};
VkPhysicalDeviceSubgroupProperties subgroup_properties = {};
VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control_properties = {};
Expand Down Expand Up @@ -935,6 +948,12 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
next_properties = &fdm_properties;
}

if (fdm_capabilities.offset_supported) {
fdmo_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_DENSITY_MAP_OFFSET_PROPERTIES_QCOM;
fdmo_properties.pNext = next_properties;
next_properties = &fdmo_properties;
}

physical_device_properties_2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
physical_device_properties_2.pNext = next_properties;
functions.GetPhysicalDeviceProperties2(physical_device, &physical_device_properties_2);
Expand Down Expand Up @@ -1002,6 +1021,17 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
print_verbose("- Vulkan Fragment Density Map not supported");
}

if (fdm_capabilities.offset_supported) {
print_verbose("- Vulkan Fragment Density Map Offset supported");

fdm_capabilities.offset_granularity.x = fdmo_properties.fragmentDensityOffsetGranularity.width;
fdm_capabilities.offset_granularity.y = fdmo_properties.fragmentDensityOffsetGranularity.height;

print_verbose(vformat(" Offset granularity: (%d, %d)", fdm_capabilities.offset_granularity.x, fdm_capabilities.offset_granularity.y));
} else {
print_verbose("- Vulkan Fragment Density Map Offset not supported");
}

if (multiview_capabilities.is_supported) {
multiview_capabilities.max_view_count = multiview_properties.maxMultiviewViewCount;
multiview_capabilities.max_instance_count = multiview_properties.maxMultiviewInstanceIndex;
Expand Down Expand Up @@ -3708,11 +3738,6 @@ RDD::FramebufferID RenderingDeviceDriverVulkan::framebuffer_create(RenderPassID
for (uint32_t i = 0; i < p_attachments.size(); i++) {
const TextureInfo *texture = (const TextureInfo *)p_attachments[i].id;
vk_img_views[i] = texture->vk_view;

if (render_pass->uses_fragment_density_map_offsets && (texture->vk_create_info.usage & VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT)) {
// If the render pass uses the FDM and the usage fits, we store the amount of layers to use it later on the render pass's end.
fragment_density_map_offsets_layers = texture->vk_create_info.arrayLayers;
}
}

VkFramebufferCreateInfo framebuffer_create_info = {};
Expand Down Expand Up @@ -4921,6 +4946,7 @@ RDD::RenderPassID RenderingDeviceDriverVulkan::render_pass_create(VectorView<Att

RenderPassInfo *render_pass = VersatileResource::allocate<RenderPassInfo>(resources_allocator);
render_pass->vk_render_pass = vk_render_pass;
render_pass->uses_fragment_density_map = uses_fragment_density_map;
return RenderPassID(render_pass);
}

Expand Down Expand Up @@ -4982,7 +5008,25 @@ void RenderingDeviceDriverVulkan::command_end_render_pass(CommandBufferID p_cmd_
DEV_ASSERT(command_buffer->active_framebuffer != nullptr && "A framebuffer must be active.");
DEV_ASSERT(command_buffer->active_render_pass != nullptr && "A render pass must be active.");

vkCmdEndRenderPass(command_buffer->vk_command_buffer);
if (device_functions.EndRenderPass2KHR != nullptr && fdm_capabilities.offset_supported && command_buffer->active_render_pass->uses_fragment_density_map) {
LocalVector<VkOffset2D> fragment_density_offsets;
if (VulkanHooks::get_singleton() != nullptr) {
fragment_density_offsets = VulkanHooks::get_singleton()->get_fragment_density_offsets();
}

VkSubpassFragmentDensityMapOffsetEndInfoQCOM offset_info = {};
offset_info.sType = VK_STRUCTURE_TYPE_SUBPASS_FRAGMENT_DENSITY_MAP_OFFSET_END_INFO_QCOM;
offset_info.pFragmentDensityOffsets = fragment_density_offsets.is_empty() ? nullptr : fragment_density_offsets.ptr();
offset_info.fragmentDensityOffsetCount = fragment_density_offsets.size();

VkSubpassEndInfo subpass_end_info = {};
subpass_end_info.sType = VK_STRUCTURE_TYPE_SUBPASS_END_INFO;
subpass_end_info.pNext = &offset_info;

device_functions.EndRenderPass2KHR(command_buffer->vk_command_buffer, &subpass_end_info);
} else {
vkCmdEndRenderPass(command_buffer->vk_command_buffer);
}

command_buffer->active_render_pass = nullptr;
command_buffer->active_framebuffer = nullptr;
Expand Down
2 changes: 1 addition & 1 deletion drivers/vulkan/rendering_device_driver_vulkan.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ class RenderingDeviceDriverVulkan : public RenderingDeviceDriver {

struct RenderPassInfo {
VkRenderPass vk_render_pass = VK_NULL_HANDLE;
bool uses_fragment_density_map_offsets = false;
bool uses_fragment_density_map = false;
};

public:
Expand Down
2 changes: 2 additions & 0 deletions drivers/vulkan/vulkan_hooks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

#pragma once

#include "core/templates/local_vector.h"
#include "drivers/vulkan/godot_vulkan.h"

class VulkanHooks {
Expand All @@ -43,5 +44,6 @@ class VulkanHooks {
virtual bool get_physical_device(VkPhysicalDevice *r_device) = 0;
virtual bool create_vulkan_device(const VkDeviceCreateInfo *p_device_create_info, VkDevice *r_device) = 0;
virtual void set_direct_queue_family_and_index(uint32_t p_queue_family_index, uint32_t p_queue_index) = 0;
virtual LocalVector<VkOffset2D> get_fragment_density_offsets() = 0;
static VulkanHooks *get_singleton() { return singleton; }
};
64 changes: 63 additions & 1 deletion modules/openxr/extensions/openxr_fb_foveation_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

#include "openxr_fb_foveation_extension.h"
#include "core/config/project_settings.h"
#include "openxr_eye_gaze_interaction.h"

#include "../openxr_platform_inc.h"

Expand All @@ -54,6 +55,14 @@ OpenXRFBFoveationExtension::OpenXRFBFoveationExtension(const String &p_rendering
swapchain_create_info_foveation_fb.next = nullptr;
swapchain_create_info_foveation_fb.flags = 0;

meta_foveation_eye_tracked_create_info.type = XR_TYPE_FOVEATION_EYE_TRACKED_PROFILE_CREATE_INFO_META;
meta_foveation_eye_tracked_create_info.next = nullptr;
meta_foveation_eye_tracked_create_info.flags = 0;

meta_foveation_eye_tracked_properties.type = XR_TYPE_SYSTEM_FOVEATION_EYE_TRACKED_PROPERTIES_META;
meta_foveation_eye_tracked_properties.next = nullptr;
meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked = XR_FALSE;

if (rendering_driver == "opengl3") {
swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB;
} else if (rendering_driver == "vulkan") {
Expand All @@ -75,6 +84,7 @@ HashMap<String, bool *> OpenXRFBFoveationExtension::get_requested_extensions() {
#ifdef XR_USE_GRAPHICS_API_VULKAN
if (rendering_driver == "vulkan") {
request_extensions[XR_FB_FOVEATION_VULKAN_EXTENSION_NAME] = &fb_foveation_vulkan_ext;
request_extensions[XR_META_FOVEATION_EYE_TRACKED_EXTENSION_NAME] = &meta_foveation_eye_tracked;
}
#endif // XR_USE_GRAPHICS_API_VULKAN

Expand All @@ -90,11 +100,16 @@ void OpenXRFBFoveationExtension::on_instance_created(const XrInstance p_instance
if (fb_foveation_configuration_ext) {
// nothing to register here...
}

if (meta_foveation_eye_tracked) {
EXT_INIT_XR_FUNC(xrGetFoveationEyeTrackedStateMETA);
}
}

void OpenXRFBFoveationExtension::on_instance_destroyed() {
fb_foveation_ext = false;
fb_foveation_configuration_ext = false;
meta_foveation_eye_tracked = false;
}

bool OpenXRFBFoveationExtension::is_enabled() const {
Expand All @@ -107,6 +122,16 @@ bool OpenXRFBFoveationExtension::is_enabled() const {
return enabled;
}

void *OpenXRFBFoveationExtension::set_system_properties_and_get_next_pointer(void *p_next_pointer) {
#ifdef XR_USE_GRAPHICS_API_VULKAN
if (rendering_driver == "vulkan") {
meta_foveation_eye_tracked_properties.next = p_next_pointer;
return &meta_foveation_eye_tracked_properties;
}
#endif
return p_next_pointer;
}

void *OpenXRFBFoveationExtension::set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) {
if (is_enabled()) {
swapchain_create_info_foveation_fb.next = p_next_pointer;
Expand Down Expand Up @@ -142,6 +167,37 @@ void OpenXRFBFoveationExtension::set_foveation_dynamic(XrFoveationDynamicFB p_fo
update_profile();
}

LocalVector<Vector2i> OpenXRFBFoveationExtension::get_fragment_density_offsets() {
LocalVector<Vector2i> ret;
if (!is_enabled() || !meta_foveation_eye_tracked || !meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked || !OpenXREyeGazeInteractionExtension::get_singleton()->is_available()) {
return ret;
}

OpenXRAPI *openxr_api = OpenXRAPI::get_singleton();
ERR_FAIL_NULL_V(openxr_api, ret);

XrFoveationEyeTrackedStateMETA state = {
XR_TYPE_FOVEATION_EYE_TRACKED_STATE_META, // type
nullptr, // next
{ XrVector2f{}, XrVector2f{} }, // foveationCenter[XR_FOVEATION_CENTER_SIZE_META];
0, // flags
};
XrResult result = xrGetFoveationEyeTrackedStateMETA(openxr_api->get_session(), &state);
if (XR_FAILED(result)) {
print_line("OpenXR: Unable to get foveation offsets [", openxr_api->get_error_string(result), "]");
return ret;
}

ret.reserve(XR_FOVEATION_CENTER_SIZE_META);
Size2 dims = openxr_api->get_recommended_target_size() * 0.5 / openxr_api->get_render_target_size_multiplier();
for (uint32_t i = 0; i < XR_FOVEATION_CENTER_SIZE_META; ++i) {
const XrVector2f &xr_offset = state.foveationCenter[i];
ret.push_back(Vector2i((int)(xr_offset.x * dims.x), (int)(xr_offset.y * dims.y)));
}

return ret;
}

void OpenXRFBFoveationExtension::_update_profile() {
// Must be called from rendering thread!
ERR_NOT_ON_RENDER_THREAD;
Expand All @@ -162,9 +218,15 @@ void OpenXRFBFoveationExtension::_update_profile() {
return;
}

void *next = nullptr;
if (fov_ext->meta_foveation_eye_tracked && fov_ext->meta_foveation_eye_tracked_properties.supportsFoveationEyeTracked) {
fov_ext->meta_foveation_eye_tracked_create_info.next = next;
next = &fov_ext->meta_foveation_eye_tracked_create_info;
}

XrFoveationLevelProfileCreateInfoFB level_profile_create_info;
level_profile_create_info.type = XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB;
level_profile_create_info.next = nullptr;
level_profile_create_info.next = next;
level_profile_create_info.level = fov_ext->foveation_level;
level_profile_create_info.verticalOffset = 0.0f;
level_profile_create_info.dynamic = fov_ext->foveation_dynamic;
Expand Down
9 changes: 9 additions & 0 deletions modules/openxr/extensions/openxr_fb_foveation_extension.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
virtual void on_instance_created(const XrInstance p_instance) override;
virtual void on_instance_destroyed() override;

virtual void *set_system_properties_and_get_next_pointer(void *p_next_pointer) override;
virtual void *set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) override;

virtual void on_main_swapchains_created() override;
Expand All @@ -69,6 +70,8 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
XrFoveationDynamicFB get_foveation_dynamic() const;
void set_foveation_dynamic(XrFoveationDynamicFB p_foveation_dynamic);

LocalVector<Vector2i> get_fragment_density_offsets();

private:
static OpenXRFBFoveationExtension *singleton;

Expand All @@ -77,6 +80,7 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
bool fb_foveation_ext = false;
bool fb_foveation_configuration_ext = false;
bool fb_foveation_vulkan_ext = false;
bool meta_foveation_eye_tracked = false;

// Configuration
XrFoveationLevelFB foveation_level = XR_FOVEATION_LEVEL_NONE_FB;
Expand All @@ -96,7 +100,12 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
XrSwapchainCreateInfoFoveationFB swapchain_create_info_foveation_fb;
OpenXRFBUpdateSwapchainExtension *swapchain_update_state_ext = nullptr;

// Enable eye tracked foveation
XrSystemFoveationEyeTrackedPropertiesMETA meta_foveation_eye_tracked_properties;
XrFoveationEyeTrackedProfileCreateInfoMETA meta_foveation_eye_tracked_create_info;

// OpenXR API call wrappers
EXT_PROTO_XRRESULT_FUNC3(xrCreateFoveationProfileFB, (XrSession), session, (const XrFoveationProfileCreateInfoFB *), create_info, (XrFoveationProfileFB *), profile);
EXT_PROTO_XRRESULT_FUNC1(xrDestroyFoveationProfileFB, (XrFoveationProfileFB), profile);
EXT_PROTO_XRRESULT_FUNC2(xrGetFoveationEyeTrackedStateMETA, (XrSession), session, (XrFoveationEyeTrackedStateMETA *), foveationState);
};
12 changes: 1 addition & 11 deletions modules/openxr/extensions/openxr_fb_update_swapchain_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,17 +105,7 @@ void OpenXRFBUpdateSwapchainExtension::on_instance_destroyed() {
}

bool OpenXRFBUpdateSwapchainExtension::is_enabled() const {
if (rendering_driver == "vulkan") {
return fb_swapchain_update_state_ext && fb_swapchain_update_state_vulkan_ext;
} else if (rendering_driver == "opengl3") {
#ifdef XR_USE_GRAPHICS_API_OPENGL_ES
return fb_swapchain_update_state_ext && fb_swapchain_update_state_opengles_ext;
#else
return fb_swapchain_update_state_ext;
#endif
}

return false;
return fb_swapchain_update_state_ext;
}

bool OpenXRFBUpdateSwapchainExtension::is_android_ext_enabled() const {
Expand Down
17 changes: 17 additions & 0 deletions modules/openxr/extensions/platform/openxr_vulkan_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,23 @@ void OpenXRVulkanExtension::set_direct_queue_family_and_index(uint32_t p_queue_f
vulkan_queue_index = p_queue_index;
}

LocalVector<VkOffset2D> OpenXRVulkanExtension::get_fragment_density_offsets() {
LocalVector<VkOffset2D> ret;
OpenXRFBFoveationExtension *fb_foveation = OpenXRFBFoveationExtension::get_singleton();
if (fb_foveation == nullptr) {
return ret;
}

LocalVector<Vector2i> offsets = fb_foveation->get_fragment_density_offsets();

ret.reserve(offsets.size());
for (const Vector2i &offset : offsets) {
ret.push_back(VkOffset2D{ offset.x, offset.y });
}

return ret;
}

XrGraphicsBindingVulkanKHR OpenXRVulkanExtension::graphics_binding_vulkan;

void *OpenXRVulkanExtension::set_session_create_and_get_next_pointer(void *p_next_pointer) {
Expand Down
1 change: 1 addition & 0 deletions modules/openxr/extensions/platform/openxr_vulkan_extension.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ class OpenXRVulkanExtension : public OpenXRGraphicsExtensionWrapper, VulkanHooks
virtual bool get_physical_device(VkPhysicalDevice *r_device) override final;
virtual bool create_vulkan_device(const VkDeviceCreateInfo *p_device_create_info, VkDevice *r_device) override final;
virtual void set_direct_queue_family_and_index(uint32_t p_queue_family_index, uint32_t p_queue_index) override final;
virtual LocalVector<VkOffset2D> get_fragment_density_offsets() override final;

virtual void get_usable_swapchain_formats(Vector<int64_t> &p_usable_swap_chains) override;
virtual void get_usable_depth_formats(Vector<int64_t> &p_usable_swap_chains) override;
Expand Down
Loading