From bc2133d685c35ff2041aef072fcf6efb2317321c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jean-Philippe=20Bruy=C3=A8re?= Date: Mon, 21 Mar 2022 12:57:52 +0100 Subject: [PATCH] timeline sync for surface operations, surface local cmd and cmdpool, surface fence sync if timeline disabled --- src/vkvg_context_internal.c | 6 ++-- src/vkvg_device.c | 2 +- src/vkvg_device_internal.c | 29 +---------------- src/vkvg_device_internal.h | 3 -- src/vkvg_fonts.c | 5 +-- src/vkvg_surface.c | 64 +++++++++++++++++++++++++------------ src/vkvg_surface_internal.c | 59 +++++++++++++++++++++++++++------- src/vkvg_surface_internal.h | 8 +++++ 8 files changed, 108 insertions(+), 68 deletions(-) diff --git a/src/vkvg_context_internal.c b/src/vkvg_context_internal.c index 68684cb..1aa5e79 100644 --- a/src/vkvg_context_internal.c +++ b/src/vkvg_context_internal.c @@ -445,7 +445,7 @@ bool _wait_and_submit_cmd (VkvgContext ctx){ #ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE VkvgSurface surf = ctx->pSurf; VkvgDevice dev = surf->dev; - vkh_timeline_wait ((VkhDevice)dev, surf->timeline, surf->timelineStep); + //vkh_timeline_wait ((VkhDevice)dev, surf->timeline, ct->timelineStep); LOCK_SURFACE(surf) LOCK_DEVICE vkh_cmd_submit_timelined (dev->gQueue, &ctx->cmd, surf->timeline, surf->timelineStep, surf->timelineStep+1); @@ -457,7 +457,7 @@ bool _wait_and_submit_cmd (VkvgContext ctx){ if (!_wait_flush_fence (ctx)) return false; - _device_reset_fence(ctx->dev, ctx->flushFence); + ResetFences (ctx->dev->vkDev, 1, &ctx->flushFence); _device_submit_cmd (ctx->dev, &ctx->cmd, ctx->flushFence); #endif @@ -889,7 +889,7 @@ void _release_context_ressources (VkvgContext ctx) { VkDevice dev = ctx->dev->vkDev; #ifndef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE - _device_destroy_fence (ctx->dev, ctx->flushFence); + vkDestroyFence (dev, ctx->flushFence, NULL); #endif vkFreeCommandBuffers(dev, ctx->cmdPool, 2, ctx->cmdBuffers); diff --git a/src/vkvg_device.c b/src/vkvg_device.c index a06bd8e..1527cda 100644 --- a/src/vkvg_device.c +++ b/src/vkvg_device.c @@ -413,7 +413,7 @@ void vkvg_device_destroy (VkvgDevice dev) vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); vkDestroyFence (dev->vkDev, dev->fence,NULL); - //vkFreeCommandBuffers (dev->vkDev, dev->cmdPool, 1, &dev->cmd); + vkFreeCommandBuffers (dev->vkDev, dev->cmdPool, 1, &dev->cmd); vkDestroyCommandPool (dev->vkDev, dev->cmdPool, NULL); vkh_queue_destroy(dev->gQueue); diff --git a/src/vkvg_device_internal.c b/src/vkvg_device_internal.c index 29709aa..6096297 100644 --- a/src/vkvg_device_internal.c +++ b/src/vkvg_device_internal.c @@ -415,29 +415,9 @@ void _device_wait_idle (VkvgDevice dev) { } void _device_wait_and_reset_device_fence (VkvgDevice dev) { vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); - _device_reset_fence(dev, dev->fence); + ResetFences (dev->vkDev, 1, &dev->fence); } -void _device_destroy_fence (VkvgDevice dev, VkFence fence) { - LOCK_DEVICE - - if (dev->gQLastFence == fence) - dev->gQLastFence = VK_NULL_HANDLE; - - vkDestroyFence (dev->vkDev, fence, NULL); - - UNLOCK_DEVICE -} -void _device_reset_fence (VkvgDevice dev, VkFence fence){ - LOCK_DEVICE - - if (dev->gQLastFence == fence) - dev->gQLastFence = VK_NULL_HANDLE; - - ResetFences (dev->vkDev, 1, &fence); - - UNLOCK_DEVICE -} bool _device_try_get_cached_context (VkvgDevice dev, VkvgContext* pCtx) { LOCK_DEVICE @@ -455,10 +435,6 @@ void _device_store_context (VkvgContext ctx) { LOCK_DEVICE -#ifndef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE - if (dev->gQLastFence == ctx->flushFence) - dev->gQLastFence = VK_NULL_HANDLE; -#endif dev->cachedContext[dev->cachedContextCount++] = ctx; ctx->references++; @@ -466,10 +442,7 @@ void _device_store_context (VkvgContext ctx) { } void _device_submit_cmd (VkvgDevice dev, VkCommandBuffer* cmd, VkFence fence) { LOCK_DEVICE - if (dev->gQLastFence != VK_NULL_HANDLE) - WaitForFences (dev->vkDev, 1, &dev->gQLastFence, VK_TRUE, VKVG_FENCE_TIMEOUT); vkh_cmd_submit (dev->gQueue, cmd, fence); - dev->gQLastFence = fence; UNLOCK_DEVICE } diff --git a/src/vkvg_device_internal.h b/src/vkvg_device_internal.h index dc44fd3..b282094 100644 --- a/src/vkvg_device_internal.h +++ b/src/vkvg_device_internal.h @@ -68,7 +68,6 @@ typedef struct _vkvg_device_t{ mtx_t mutex; /**< protect device access (queue, cahes, ...)from ctxs in separate threads */ bool threadAware; /**< if true, mutex is created and guard device queue and caches access */ VkhQueue gQueue; /**< Vulkan Queue with Graphic flag */ - VkFence gQLastFence; VkRenderPass renderPass; /**< Vulkan render pass, common for all surfaces */ VkRenderPass renderPass_ClearStencil;/**< Vulkan render pass for first draw with context, stencil has to be cleared */ @@ -139,8 +138,6 @@ void _device_wait_idle (VkvgDevice dev); void _device_wait_and_reset_device_fence(VkvgDevice dev); void _device_submit_cmd (VkvgDevice dev, VkCommandBuffer* cmd, VkFence fence); -void _device_destroy_fence (VkvgDevice dev, VkFence fence); -void _device_reset_fence (VkvgDevice dev, VkFence fence); bool _device_try_get_cached_context (VkvgDevice dev, VkvgContext* pCtx); void _device_store_context (VkvgContext ctx); #endif diff --git a/src/vkvg_fonts.c b/src/vkvg_fonts.c index efb2c9a..830d9f3 100644 --- a/src/vkvg_fonts.c +++ b/src/vkvg_fonts.c @@ -105,7 +105,7 @@ void _increase_font_tex_array (VkvgDevice dev){ _font_cache_t* cache = dev->fontCache; vkWaitForFences (dev->vkDev, 1, &cache->uploadFence, VK_TRUE, UINT64_MAX); - _device_reset_fence (dev, cache->uploadFence); + ResetFences (dev->vkDev, 1, &cache->uploadFence); vkResetCommandBuffer(cache->cmd, 0); @@ -167,7 +167,8 @@ void _flush_chars_to_tex (VkvgDevice dev, _vkvg_font_t* f) { LOG(VKVG_LOG_INFO, "_flush_chars_to_tex pen(%d, %d)\n",f->curLine.penX, f->curLine.penY); vkWaitForFences (dev->vkDev,1,&cache->uploadFence,VK_TRUE,UINT64_MAX); - _device_reset_fence (dev, cache->uploadFence); + ResetFences (dev->vkDev, 1, &cache->uploadFence); + vkResetCommandBuffer(cache->cmd,0); memcpy(cache->buff.allocInfo.pMappedData, cache->hostBuff, (uint64_t)f->curLine.height * FONT_PAGE_SIZE * cache->texPixelSize); diff --git a/src/vkvg_surface.c b/src/vkvg_surface.c index 76b4dbc..3e6cea7 100644 --- a/src/vkvg_surface.c +++ b/src/vkvg_surface.c @@ -30,6 +30,31 @@ #include "vkh_image.h" #define max(x,y) +void _transition_surf_images (VkvgSurface surf) { + LOCK_SURFACE(surf) + VkvgDevice dev = surf->dev; + + //_surface_wait_cmd (surf); + + vkh_cmd_begin (surf->cmd,VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); + VkhImage imgMs = surf->imgMS; + if (imgMs != NULL) + vkh_image_set_layout(surf->cmd, imgMs, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + + vkh_image_set_layout(surf->cmd, surf->img, VK_IMAGE_ASPECT_COLOR_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + vkh_image_set_layout (surf->cmd, surf->stencil, VK_IMAGE_ASPECT_STENCIL_BIT, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT); + vkh_cmd_end (surf->cmd); + + _surface_submit_cmd (surf); + + UNLOCK_SURFACE(surf) +} void vkvg_surface_clear (VkvgSurface surf) { if (surf->status) return; @@ -46,6 +71,8 @@ VkvgSurface vkvg_surface_create (VkvgDevice dev, uint32_t width, uint32_t height _create_surface_images (surf); + _transition_surf_images (surf); + surf->status = VKVG_STATUS_SUCCESS; vkvg_device_reference (surf->dev); return surf; @@ -71,7 +98,9 @@ VkvgSurface vkvg_surface_create_for_VkhImage (VkvgDevice dev, void* vkhImg) { _create_surface_secondary_images (surf); _create_framebuffer (surf); - _clear_surface (surf, VK_IMAGE_ASPECT_STENCIL_BIT); + + _transition_surf_images (surf); + //_clear_surface (surf, VK_IMAGE_ASPECT_STENCIL_BIT); surf->status = VKVG_STATUS_SUCCESS; vkvg_device_reference (surf->dev); @@ -110,9 +139,7 @@ VkvgSurface vkvg_surface_create_from_bitmap (VkvgDevice dev, unsigned char* img, memcpy (buff.allocInfo.pMappedData, img, imgSize); - VkCommandBuffer cmd = dev->cmd; - - _device_wait_and_reset_device_fence (dev); + VkCommandBuffer cmd = surf->cmd; vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT, @@ -148,10 +175,8 @@ VkvgSurface vkvg_surface_create_from_bitmap (VkvgDevice dev, unsigned char* img, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT); vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); - //don't reset fence after completion as this is the last cmd. (signaled idle fence) - vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); + _surface_submit_cmd (surf); vkvg_buffer_destroy (&buff); vkh_image_destroy (stagImg); @@ -212,6 +237,7 @@ void vkvg_surface_destroy(VkvgSurface surf) } UNLOCK_SURFACE(surf) + vkDestroyCommandPool(surf->dev->vkDev, surf->cmdPool, NULL); vkDestroyFramebuffer(surf->dev->vkDev, surf->fb, NULL); if (!surf->img->imported) @@ -225,6 +251,8 @@ void vkvg_surface_destroy(VkvgSurface surf) #if VKVG_ENABLE_VK_TIMELINE_SEMAPHORE vkDestroySemaphore (surf->dev->vkDev, surf->timeline, NULL); +#else + vkDestroyFence (surf->dev->vkDev, surf->flushFence, NULL); #endif vkvg_device_destroy (surf->dev); @@ -308,9 +336,7 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){ VMA_MEMORY_USAGE_GPU_ONLY, VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); - VkCommandBuffer cmd = dev->cmd; - _device_wait_and_reset_device_fence (dev); - + VkCommandBuffer cmd = surf->cmd; vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, @@ -330,7 +356,8 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){ vkh_image_get_vkimage (stagImg), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST); vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); + + _surface_submit_cmd (surf); VkhImage stagImgLinear = stagImg; @@ -345,7 +372,6 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){ .dstOffset = {0}, .extent = {(int32_t)surf->width, (int32_t)surf->height, 1} }; - _device_wait_and_reset_device_fence (dev); vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); vkh_image_set_layout (cmd, stagImgLinear, VK_IMAGE_ASPECT_COLOR_BIT, @@ -359,12 +385,11 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){ vkh_image_get_vkimage (stagImgLinear), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy); vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); - vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); + _surface_submit_cmd (surf); + vkh_image_destroy (stagImg); - } else - vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); + } void* img = vkh_image_map (stagImgLinear); @@ -399,8 +424,7 @@ vkvg_status_t vkvg_surface_write_to_memory (VkvgSurface surf, unsigned char* con VMA_MEMORY_USAGE_GPU_TO_CPU, VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); - VkCommandBuffer cmd = dev->cmd; - _device_wait_and_reset_device_fence (dev); + VkCommandBuffer cmd = surf->cmd; vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT, @@ -421,8 +445,8 @@ vkvg_status_t vkvg_surface_write_to_memory (VkvgSurface surf, unsigned char* con vkh_image_get_vkimage (stagImg), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST); vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); - vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX); + + _surface_submit_cmd (surf); uint64_t stride = vkh_image_get_stride(stagImg); uint32_t dest_stride = surf->width * 4; diff --git a/src/vkvg_surface_internal.c b/src/vkvg_surface_internal.c index 8c575c3..4ef6668 100644 --- a/src/vkvg_surface_internal.c +++ b/src/vkvg_surface_internal.c @@ -20,15 +20,13 @@ * THE SOFTWARE. */ -#include "vkvg_surface_internal.h" #include "vkvg_device_internal.h" +#include "vkvg_surface_internal.h" #include "vkh_image.h" +#include "vkh_queue.h" void _explicit_ms_resolve (VkvgSurface surf){ - VkvgDevice dev = surf->dev; - VkCommandBuffer cmd = dev->cmd; - - _device_wait_and_reset_device_fence (dev); + VkCommandBuffer cmd = surf->cmd; vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); vkh_image_set_layout (cmd, surf->imgMS, VK_IMAGE_ASPECT_COLOR_BIT, @@ -53,15 +51,12 @@ void _explicit_ms_resolve (VkvgSurface surf){ VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT); vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); + _surface_submit_cmd (surf); } void _clear_surface (VkvgSurface surf, VkImageAspectFlags aspect) { - VkvgDevice dev = surf->dev; - VkCommandBuffer cmd = dev->cmd; - - _device_wait_and_reset_device_fence (dev); + VkCommandBuffer cmd = surf->cmd; vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); @@ -101,7 +96,7 @@ void _clear_surface (VkvgSurface surf, VkImageAspectFlags aspect) } vkh_cmd_end (cmd); - _device_submit_cmd (dev, &cmd, dev->fence); + _surface_submit_cmd (surf); } void _create_surface_main_image (VkvgSurface surf){ @@ -186,9 +181,51 @@ VkvgSurface _create_surface (VkvgDevice dev, VkFormat format) { surf->format = format; if (dev->threadAware) mtx_init (&surf->mutex, mtx_plain); + surf->cmdPool = vkh_cmd_pool_create ((VkhDevice)dev, dev->gQueue->familyIndex, VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT); + vkh_cmd_buffs_create((VkhDevice)dev, surf->cmdPool,VK_COMMAND_BUFFER_LEVEL_PRIMARY, 1, &surf->cmd); #if VKVG_ENABLE_VK_TIMELINE_SEMAPHORE surf->timeline = vkh_timeline_create ((VkhDevice)dev, 0); +#else + surf->flushFence = vkh_fence_create ((VkhDevice)dev); +#endif + +#if defined(DEBUG) && defined(VKVG_DBG_UTILS) + vkh_device_set_object_name((VkhDevice)dev, VK_OBJECT_TYPE_COMMAND_BUFFER, (uint64_t)surf->cmd, "vkvgSurfCmd"); #endif + return surf; } +//if fence sync, surf mutex must be locked. +/*bool _surface_wait_cmd (VkvgSurface surf) { + LOG(VKVG_LOG_INFO, "SURF: _surface__wait_flush_fence\n"); +#ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE + if (vkh_timeline_wait ((VkhDevice)surf->dev, surf->timeline, surf->timelineStep) == VK_SUCCESS) + return true; +#else + if (WaitForFences (surf->dev->vkDev, 1, &surf->flushFence, VK_TRUE, VKVG_FENCE_TIMEOUT) == VK_SUCCESS) { + ResetFences (surf->dev->vkDev, 1, &surf->flushFence); + return true; + } +#endif + LOG(VKVG_LOG_DEBUG, "CTX: _wait_flush_fence timeout\n"); + surf->status = VKVG_STATUS_TIMEOUT; + return false; +}*/ +//surface mutex must be locked to call this method, locking to guard also the surf->cmd local buffer usage. +void _surface_submit_cmd (VkvgSurface surf) { + VkvgDevice dev = surf->dev; +#ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE + LOCK_DEVICE + vkh_cmd_submit_timelined (dev->gQueue, &surf->cmd, surf->timeline, surf->timelineStep, surf->timelineStep+1); + surf->timelineStep++; + UNLOCK_DEVICE + vkh_timeline_wait ((VkhDevice)dev, surf->timeline, surf->timelineStep); +#else + LOCK_DEVICE + vkh_cmd_submit (surf->dev->gQueue, &surf->cmd, surf->flushFence); + UNLOCK_DEVICE + WaitForFences (surf->dev->vkDev, 1, &surf->flushFence, VK_TRUE, VKVG_FENCE_TIMEOUT); + ResetFences (surf->dev->vkDev, 1, &surf->flushFence); +#endif +} diff --git a/src/vkvg_surface_internal.h b/src/vkvg_surface_internal.h index 0b52d4b..5cc47c6 100644 --- a/src/vkvg_surface_internal.h +++ b/src/vkvg_surface_internal.h @@ -37,12 +37,17 @@ typedef struct _vkvg_surface_t { VkhImage img; VkhImage imgMS; VkhImage stencil; + VkCommandPool cmdPool; //local pools ensure thread safety + VkCommandBuffer cmd; //surface local command buffer. bool newSurf; mtx_t mutex; #ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE VkSemaphore timeline; /**< Timeline semaphore */ uint64_t timelineStep; +#else + VkFence flushFence; //unsignaled idle. #endif + }vkvg_surface; #define LOCK_SURFACE(surf) \ @@ -59,4 +64,7 @@ void _create_surface_secondary_images (VkvgSurface surf); void _create_framebuffer (VkvgSurface surf); void _create_surface_images (VkvgSurface surf); VkvgSurface _create_surface (VkvgDevice dev, VkFormat format); + +void _surface_submit_cmd (VkvgSurface surf); +//bool _surface_wait_cmd (VkvgSurface surf); #endif -- 2.47.3