fix multiple validation layer issues

#33

Engine/gapi/spirv/meshconverter.cpp

@@ -73,17 +73,19 @@ void MeshConverter::exec() {
   // gl_MeshPerVertexNV block
   for(auto it = code.begin(), end = code.end(); it!=end; ++it) {
     auto& i = *it;
-    if(i.op()==spv::OpMemberDecorate && i[3]==spv::DecorationPerViewNV) {
-      idMeshPerVertexNV = i[1];
+    if(i.op()==spv::OpMemberDecorate && i[3]==spv::DecorationBuiltIn && i[4]==spv::BuiltInPosition) {
+      idMeshPerVertexNV              = i[1];
+      gl_MeshPerVertexNV.gl_Position = true;
       break;
       }
     }
 
   removeMultiview(code);
   removeCullClip(code);
+
   for(auto it = code.begin(), end = code.end(); it!=end; ++it) {
     auto& i = *it;
-    if(i.op()==spv::OpDecorate && i[2]==spv::DecorationBlock) {
+    if(i.op()==spv::OpDecorate && i[1]==idMeshPerVertexNV && i[2]==spv::DecorationBlock) {
       it.setToNop();
       continue;
       }
@@ -93,6 +95,9 @@ void MeshConverter::exec() {
       continue;
       }
     if(i.op()==spv::OpMemberDecorate && i[3]==spv::DecorationBuiltIn) {
+      if(i[4]==spv::BuiltInPointSize) {
+        gl_MeshPerVertexNV.gl_PointSize = true;
+        }
       it.setToNop();
       continue;
       }
@@ -106,6 +111,12 @@ void MeshConverter::exec() {
         idGlPerVertex = i[1];
       continue;
       }
+    if(i.op()==spv::OpSourceExtension) {
+      auto str = std::string_view(reinterpret_cast<const char*>(&i[2]));
+      if(str=="GL_NV_mesh_shader")
+        it.setToNop();
+      continue;
+      }
     }
 
   avoidReservedFixup();
@@ -359,7 +370,9 @@ void MeshConverter::generateVs() {
 
   fn.insert(spv::OpName,       typeRemaps[idGlPerVertex], "gl_PerVertex");
   fn.insert(spv::OpMemberName, typeRemaps[idGlPerVertex], 0, "gl_Position");
-  fn.insert(spv::OpMemberName, typeRemaps[idGlPerVertex], 1, "gl_PointSize");
+  if(gl_MeshPerVertexNV.gl_PointSize) {
+    fn.insert(spv::OpMemberName, typeRemaps[idGlPerVertex], 1, "gl_PointSize");
+    }
   //fn.insert(spv::OpMemberName, typeRemaps[idGlPerVertex], 2, "gl_ClipDistance");
   //fn.insert(spv::OpMemberName, typeRemaps[idGlPerVertex], 3, "gl_CullDistance");
 
@@ -379,7 +392,9 @@ void MeshConverter::generateVs() {
   fn.insert(spv::OpDecorate,       {gl_VertexIndex, spv::DecorationBuiltIn, spv::BuiltInVertexIndex});
   fn.insert(spv::OpDecorate,       {typeRemaps[idGlPerVertex], spv::DecorationBlock});
   fn.insert(spv::OpMemberDecorate, {typeRemaps[idGlPerVertex], 0, spv::DecorationBuiltIn, spv::BuiltInPosition});
-  fn.insert(spv::OpMemberDecorate, {typeRemaps[idGlPerVertex], 1, spv::DecorationBuiltIn, spv::BuiltInPointSize});
+  if(gl_MeshPerVertexNV.gl_PointSize) {
+    fn.insert(spv::OpMemberDecorate, {typeRemaps[idGlPerVertex], 1, spv::DecorationBuiltIn, spv::BuiltInPointSize});
+    }
   //fn.insert(spv::OpMemberDecorate, {typeRemaps[idGlPerVertex], 2, spv::DecorationBuiltIn, spv::BuiltInClipDistance});
   //fn.insert(spv::OpMemberDecorate, {typeRemaps[idGlPerVertex], 3, spv::DecorationBuiltIn, spv::BuiltInCullDistance});
   }

Engine/gapi/spirv/meshconverter.h

@@ -37,6 +37,11 @@ class MeshConverter {
     libspirv::MutableBytecode& code;
     libspirv::MutableBytecode  vert;
 
+    struct gl_MeshPerVertexNV {
+      bool gl_Position  = false;
+      bool gl_PointSize = false;
+      } gl_MeshPerVertexNV;
+
     // meslet builtins
     uint32_t idMeshPerVertexNV    = 0;
     uint32_t idGlPerVertex        = 0;

Engine/gapi/vulkan/vcommandbuffer.cpp

@@ -1051,7 +1051,7 @@ void VMeshCommandBuffer::setBytes(AbstractGraphicsApi::Pipeline& p, const void*
   VPipeline& px = reinterpret_cast<VPipeline&>(p);
   if(px.meshPipeline()==VK_NULL_HANDLE)
     return;
-  vkCmdPushConstants(impl, px.meshPipelineLayout(), VK_PIPELINE_BIND_POINT_COMPUTE, 0, uint32_t(size), data);
+  vkCmdPushConstants(cbHelper, px.meshPipelineLayout(), VK_SHADER_STAGE_COMPUTE_BIT, 0, uint32_t(size), data);
   }
 
 void VMeshCommandBuffer::setUniforms(AbstractGraphicsApi::Pipeline& p, AbstractGraphicsApi::Desc& u) {

Engine/gapi/vulkan/vmeshlethelper.h

@@ -28,9 +28,9 @@ class VMeshletHelper {
 
   public:
     enum {
-      PipelineMemorySize = 16*1024*1024,
-      MeshletsMemorySize = 16*1024,
-      IndirectMemorySize = 1024*sizeof(VkDrawIndexedIndirectCommand),
+      PipelineMemorySize = 32*1024*1024,
+      MeshletsMemorySize = 32*1024,
+      IndirectMemorySize = 2048*sizeof(VkDrawIndexedIndirectCommand),
       };
     explicit VMeshletHelper(VDevice& dev);
     ~VMeshletHelper();

Engine/gapi/vulkan/vpipeline.cpp

@@ -53,6 +53,10 @@ VPipeline::VPipeline(VDevice& device, const RenderState& st, Topology tp,
         defaultStride += uint32_t(Decl::size(decl[i]));
         }
       }
+
+    if(ulay.pb.size>0) {
+      pushStageFlags = nativeFormat(ulay.pb.stage);
+      }
     pipelineLayout = initLayout(device,ulay,pushStageFlags,pushSize,false);
 
     if(auto ms=findShader(ShaderReflection::Stage::Mesh)) {
@@ -69,6 +73,9 @@ VPipeline::VPipeline(VDevice& device, const RenderState& st, Topology tp,
       info.stage.pName  = "main";
       info.layout       = pipelineLayoutMs;
       vkAssert(vkCreateComputePipelines(device.device.impl, VK_NULL_HANDLE, 1, &info, nullptr, &meshCompuePipeline));
+
+      // cancel native mesh shading
+      pushStageFlags &= ~VK_SHADER_STAGE_MESH_BIT_NV;
       }
     }
   catch(...) {
@@ -153,8 +160,8 @@ void VPipeline::cleanup() {
     vkDestroyPipeline(device,i.val,nullptr);
   }
 
-VkPipelineLayout VPipeline::initLayout(VDevice& device, const VPipelineLay& uboLay,
-                                       VkShaderStageFlags& pushStageFlags, uint32_t& pushSize,
+VkPipelineLayout VPipeline::initLayout(VDevice& dev, const VPipelineLay& uboLay,
+                                       VkShaderStageFlags pushStageFlags, uint32_t& pushSize,
                                        bool isMeshCompPass) {
   VkPushConstantRange push = {};
 
@@ -168,7 +175,7 @@ VkPipelineLayout VPipeline::initLayout(VDevice& device, const VPipelineLay& uboL
 
   if(uboLay.msHelper!=VK_NULL_HANDLE) {
     if(isMeshCompPass) {
-      auto& ms = *device.meshHelper.get();
+      auto& ms = *dev.meshHelper.get();
       pSetLayouts[pipelineLayoutInfo.setLayoutCount] = ms.lay();
       pipelineLayoutInfo.setLayoutCount++;
       } else {
@@ -179,6 +186,16 @@ VkPipelineLayout VPipeline::initLayout(VDevice& device, const VPipelineLay& uboL
 
   if(uboLay.pb.size>0) {
     pushStageFlags = nativeFormat(uboLay.pb.stage);
+
+    if(uboLay.msHelper!=VK_NULL_HANDLE) {
+      if(isMeshCompPass) {
+        if((pushStageFlags&VK_SHADER_STAGE_MESH_BIT_NV)==VK_SHADER_STAGE_MESH_BIT_NV) {
+          pushStageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
+          }
+        } else {
+        pushStageFlags &= ~VK_SHADER_STAGE_MESH_BIT_NV;
+        }
+      }
     push.stageFlags = pushStageFlags;
     push.offset     = 0;
     push.size       = uint32_t(uboLay.pb.size);
@@ -190,7 +207,7 @@ VkPipelineLayout VPipeline::initLayout(VDevice& device, const VPipelineLay& uboL
     }
 
   VkPipelineLayout ret;
-  vkAssert(vkCreatePipelineLayout(device.device.impl,&pipelineLayoutInfo,nullptr,&ret));
+  vkAssert(vkCreatePipelineLayout(dev.device.impl,&pipelineLayoutInfo,nullptr,&ret));
   return ret;
   }
 

Engine/gapi/vulkan/vpipeline.h

@@ -74,7 +74,7 @@ class VPipeline : public AbstractGraphicsApi::Pipeline {
     const VShader*                         findShader(ShaderReflection::Stage sh) const;
 
     void cleanup();
-    static VkPipelineLayout      initLayout(VDevice& device, const VPipelineLay& uboLay, VkShaderStageFlags& pushFlg, uint32_t& pushSize, bool isMeshCompPass);
+    static VkPipelineLayout      initLayout(VDevice& device, const VPipelineLay& uboLay, VkShaderStageFlags pushFlg, uint32_t& pushSize, bool isMeshCompPass);
     VkPipeline                   initGraphicsPipeline(VkDevice device, VkPipelineLayout layout,
                                                       const VFramebufferMap::RenderPass* rpLay, const VkPipelineRenderingCreateInfoKHR* dynLay, const RenderState &st,
                                                       const Decl::ComponentType *decl, size_t declSize, size_t stride,

Engine/gapi/vulkan/vpipelinelay.cpp

@@ -73,8 +73,10 @@ VkDescriptorSetLayout VPipelineLay::create(uint32_t runtimeArraySz) const {
     b.descriptorCount = e.runtimeSized ? runtimeArraySz : e.arraySize;
     b.descriptorType  = nativeFormat(e.cls);
     b.stageFlags      = nativeFormat(e.stage);
-    if(b.stageFlags==VK_SHADER_STAGE_MESH_BIT_NV && dev.props.meshlets.meshShaderEmulated)
-      b.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
+    if((b.stageFlags&VK_SHADER_STAGE_MESH_BIT_NV)==VK_SHADER_STAGE_MESH_BIT_NV && dev.props.meshlets.meshShaderEmulated) {
+      b.stageFlags &= ~VK_SHADER_STAGE_MESH_BIT_NV;
+      b.stageFlags |= VK_SHADER_STAGE_COMPUTE_BIT;
+      }
     ++count;
     }
 

Engine/libspirv/libspirv.cpp

@@ -245,11 +245,14 @@ uint32_t MutableBytecode::OpTypeArray(Iterator& typesEnd, uint32_t eltType, uint
   }
 
 uint32_t MutableBytecode::OpTypeRuntimeArray(Iterator& typesEnd, uint32_t eltType) {
+  // NOTE: spirv allows for multiple declarations of array with different strides(specifyed by Decoration)
+  /*
   for(auto it=begin(); it!=typesEnd; ++it) {
     auto& i = *it;
     if(i.op()==spv::OpTypeRuntimeArray && i[2]==eltType)
-      return i[1];
+      ;//return i[1];
     }
+  */
   const uint32_t tRet = fetchAddBound();
   typesEnd.insert(spv::OpTypeRuntimeArray, {tRet, eltType});
   return tRet;