timeline sync for surface operations, surface local cmd and cmdpool, surface fence...

diff --git a/src/vkvg_context_internal.c b/src/vkvg_context_internal.c
index 68684cbeac4f9e7a6db5c8d5da462a44f806fb8b..1aa5e79715beba0e7f7946d5642c687e9b82b3a0 100644 (file)
--- a/src/vkvg_context_internal.c
+++ b/src/vkvg_context_internal.c
@@ -445,7 +445,7 @@ bool _wait_and_submit_cmd (VkvgContext ctx){
 #ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
        VkvgSurface surf = ctx->pSurf;
        VkvgDevice dev = surf->dev;
-       vkh_timeline_wait ((VkhDevice)dev, surf->timeline, surf->timelineStep);
+       //vkh_timeline_wait ((VkhDevice)dev, surf->timeline, ct->timelineStep);
        LOCK_SURFACE(surf)
        LOCK_DEVICE
        vkh_cmd_submit_timelined (dev->gQueue, &ctx->cmd, surf->timeline, surf->timelineStep, surf->timelineStep+1);
@@ -457,7 +457,7 @@ bool _wait_and_submit_cmd (VkvgContext ctx){
 
        if (!_wait_flush_fence (ctx))
                return false;
-       _device_reset_fence(ctx->dev, ctx->flushFence);
+       ResetFences (ctx->dev->vkDev, 1, &ctx->flushFence);
        _device_submit_cmd (ctx->dev, &ctx->cmd, ctx->flushFence);
 #endif
 
@@ -889,7 +889,7 @@ void _release_context_ressources (VkvgContext ctx) {
        VkDevice dev = ctx->dev->vkDev;
        
 #ifndef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
-       _device_destroy_fence (ctx->dev, ctx->flushFence);
+       vkDestroyFence (dev, ctx->flushFence, NULL);
 #endif
 
        vkFreeCommandBuffers(dev, ctx->cmdPool, 2, ctx->cmdBuffers);

diff --git a/src/vkvg_device.c b/src/vkvg_device.c
index a06bd8e3bfa0fee5f9856df5fde9ec209f5a3524..1527cdaa3f94dbb77b5d4e9afde92c248b246759 100644 (file)
--- a/src/vkvg_device.c
+++ b/src/vkvg_device.c
@@ -413,7 +413,7 @@ void vkvg_device_destroy (VkvgDevice dev)
        vkWaitForFences                                 (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
        vkDestroyFence                                  (dev->vkDev, dev->fence,NULL);
 
-       //vkFreeCommandBuffers                  (dev->vkDev, dev->cmdPool, 1, &dev->cmd);
+       vkFreeCommandBuffers                    (dev->vkDev, dev->cmdPool, 1, &dev->cmd);
        vkDestroyCommandPool                    (dev->vkDev, dev->cmdPool, NULL);
 
        vkh_queue_destroy(dev->gQueue);

diff --git a/src/vkvg_device_internal.c b/src/vkvg_device_internal.c
index 29709aa6bbafffacab65566dd186ba65918c1b03..6096297c21baddda4e7f3cf7546d968b5bd44426 100644 (file)
--- a/src/vkvg_device_internal.c
+++ b/src/vkvg_device_internal.c
@@ -415,29 +415,9 @@ void _device_wait_idle (VkvgDevice dev) {
 }
 void _device_wait_and_reset_device_fence (VkvgDevice dev) {
        vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
-       _device_reset_fence(dev, dev->fence);
+       ResetFences (dev->vkDev, 1, &dev->fence);
 }
 
-void _device_destroy_fence (VkvgDevice dev, VkFence fence) {
-       LOCK_DEVICE
-
-       if (dev->gQLastFence == fence)
-               dev->gQLastFence = VK_NULL_HANDLE;
-
-       vkDestroyFence (dev->vkDev, fence, NULL);
-
-       UNLOCK_DEVICE
-}
-void _device_reset_fence (VkvgDevice dev, VkFence fence){
-       LOCK_DEVICE
-
-       if (dev->gQLastFence == fence)
-               dev->gQLastFence = VK_NULL_HANDLE;
-
-       ResetFences (dev->vkDev, 1, &fence);
-
-       UNLOCK_DEVICE
-}
 bool _device_try_get_cached_context (VkvgDevice dev, VkvgContext* pCtx) {
        LOCK_DEVICE
 
@@ -455,10 +435,6 @@ void _device_store_context (VkvgContext ctx) {
 
        LOCK_DEVICE
 
-#ifndef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
-       if (dev->gQLastFence == ctx->flushFence)
-               dev->gQLastFence = VK_NULL_HANDLE;
-#endif
        dev->cachedContext[dev->cachedContextCount++] = ctx;
        ctx->references++;
 
@@ -466,10 +442,7 @@ void _device_store_context (VkvgContext ctx) {
 }
 void _device_submit_cmd (VkvgDevice dev, VkCommandBuffer* cmd, VkFence fence) {
        LOCK_DEVICE
-       if (dev->gQLastFence != VK_NULL_HANDLE)
-               WaitForFences (dev->vkDev, 1, &dev->gQLastFence, VK_TRUE, VKVG_FENCE_TIMEOUT);
        vkh_cmd_submit (dev->gQueue, cmd, fence);
-       dev->gQLastFence = fence;
        UNLOCK_DEVICE
 }
 

diff --git a/src/vkvg_device_internal.h b/src/vkvg_device_internal.h
index dc44fd3d1879cd1c4e9b52d366e0bf03a9546e56..b282094b17504421aa678f6315690b50bc865b6b 100644 (file)
--- a/src/vkvg_device_internal.h
+++ b/src/vkvg_device_internal.h
@@ -68,7 +68,6 @@ typedef struct _vkvg_device_t{
        mtx_t                                   mutex;                                  /**< protect device access (queue, cahes, ...)from ctxs in separate threads */
        bool                                    threadAware;                    /**< if true, mutex is created and guard device queue and caches access */
        VkhQueue                                gQueue;                                 /**< Vulkan Queue with Graphic flag */
-       VkFence                                 gQLastFence;
 
        VkRenderPass                    renderPass;                             /**< Vulkan render pass, common for all surfaces */
        VkRenderPass                    renderPass_ClearStencil;/**< Vulkan render pass for first draw with context, stencil has to be cleared */
@@ -139,8 +138,6 @@ void _device_wait_idle                                      (VkvgDevice dev);
 void _device_wait_and_reset_device_fence(VkvgDevice dev);
 void _device_submit_cmd                                        (VkvgDevice dev, VkCommandBuffer* cmd, VkFence fence);
 
-void _device_destroy_fence                             (VkvgDevice dev, VkFence fence);
-void _device_reset_fence                               (VkvgDevice dev, VkFence fence);
 bool _device_try_get_cached_context            (VkvgDevice dev, VkvgContext* pCtx);
 void _device_store_context                             (VkvgContext ctx);
 #endif

diff --git a/src/vkvg_fonts.c b/src/vkvg_fonts.c
index efb2c9a3ea40bdddf5a70bc5570a01e6427ca9de..830d9f3656d9d11eb5ee74c695749de0c005de93 100644 (file)
--- a/src/vkvg_fonts.c
+++ b/src/vkvg_fonts.c
@@ -105,7 +105,7 @@ void _increase_font_tex_array (VkvgDevice dev){
        _font_cache_t* cache = dev->fontCache;
 
        vkWaitForFences         (dev->vkDev, 1, &cache->uploadFence, VK_TRUE, UINT64_MAX);
-       _device_reset_fence (dev, cache->uploadFence);
+       ResetFences (dev->vkDev, 1, &cache->uploadFence);
 
        vkResetCommandBuffer(cache->cmd, 0);
 
@@ -167,7 +167,8 @@ void _flush_chars_to_tex (VkvgDevice dev, _vkvg_font_t* f) {
 
        LOG(VKVG_LOG_INFO, "_flush_chars_to_tex pen(%d, %d)\n",f->curLine.penX, f->curLine.penY);
        vkWaitForFences         (dev->vkDev,1,&cache->uploadFence,VK_TRUE,UINT64_MAX);
-       _device_reset_fence (dev, cache->uploadFence);
+       ResetFences (dev->vkDev, 1, &cache->uploadFence);
+
        vkResetCommandBuffer(cache->cmd,0);
 
        memcpy(cache->buff.allocInfo.pMappedData, cache->hostBuff, (uint64_t)f->curLine.height * FONT_PAGE_SIZE * cache->texPixelSize);

diff --git a/src/vkvg_surface.c b/src/vkvg_surface.c
index 76b4dbc5d2adfd84db5a132da38ff3f8b77f27bc..3e6cea76f185f336ad9077cc3f905b1e674dce4f 100644 (file)
--- a/src/vkvg_surface.c
+++ b/src/vkvg_surface.c
@@ -30,6 +30,31 @@
 #include "vkh_image.h"
 
 #define max(x,y)
+void _transition_surf_images (VkvgSurface surf) {
+       LOCK_SURFACE(surf)
+       VkvgDevice dev = surf->dev;
+
+       //_surface_wait_cmd (surf);
+
+       vkh_cmd_begin (surf->cmd,VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
+       VkhImage imgMs = surf->imgMS;
+       if (imgMs != NULL)
+               vkh_image_set_layout(surf->cmd, imgMs, VK_IMAGE_ASPECT_COLOR_BIT,
+                                                        VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+                                                        VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+
+       vkh_image_set_layout(surf->cmd, surf->img, VK_IMAGE_ASPECT_COLOR_BIT,
+                                        VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+                                        VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+       vkh_image_set_layout (surf->cmd, surf->stencil, VK_IMAGE_ASPECT_STENCIL_BIT,
+                                                 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
+                                                 VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT);
+       vkh_cmd_end (surf->cmd);
+
+       _surface_submit_cmd (surf);
+
+       UNLOCK_SURFACE(surf)
+}
 void vkvg_surface_clear (VkvgSurface surf) {
        if (surf->status)
                return;
@@ -46,6 +71,8 @@ VkvgSurface vkvg_surface_create (VkvgDevice dev, uint32_t width, uint32_t height
 
        _create_surface_images (surf);
 
+       _transition_surf_images (surf);
+
        surf->status = VKVG_STATUS_SUCCESS;
        vkvg_device_reference (surf->dev);
        return surf;
@@ -71,7 +98,9 @@ VkvgSurface vkvg_surface_create_for_VkhImage (VkvgDevice dev, void* vkhImg) {
 
        _create_surface_secondary_images        (surf);
        _create_framebuffer                                     (surf);
-       _clear_surface                                          (surf, VK_IMAGE_ASPECT_STENCIL_BIT);
+
+       _transition_surf_images (surf);
+       //_clear_surface                                                (surf, VK_IMAGE_ASPECT_STENCIL_BIT);
 
        surf->status = VKVG_STATUS_SUCCESS;
        vkvg_device_reference (surf->dev);
@@ -110,9 +139,7 @@ VkvgSurface vkvg_surface_create_from_bitmap (VkvgDevice dev, unsigned char* img,
 
        memcpy (buff.allocInfo.pMappedData, img, imgSize);      
 
-       VkCommandBuffer cmd = dev->cmd;
-
-       _device_wait_and_reset_device_fence (dev);
+       VkCommandBuffer cmd = surf->cmd;
 
        vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
        vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT,
@@ -148,10 +175,8 @@ VkvgSurface vkvg_surface_create_from_bitmap (VkvgDevice dev, unsigned char* img,
                                                  VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
 
        vkh_cmd_end             (cmd);
-       _device_submit_cmd              (dev, &cmd, dev->fence);
 
-       //don't reset fence after completion as this is the last cmd. (signaled idle fence)
-       vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
+       _surface_submit_cmd (surf);
 
        vkvg_buffer_destroy (&buff);
        vkh_image_destroy       (stagImg);
@@ -212,6 +237,7 @@ void vkvg_surface_destroy(VkvgSurface surf)
        }
        UNLOCK_SURFACE(surf)
 
+       vkDestroyCommandPool(surf->dev->vkDev, surf->cmdPool, NULL);
        vkDestroyFramebuffer(surf->dev->vkDev, surf->fb, NULL);
 
        if (!surf->img->imported)
@@ -225,6 +251,8 @@ void vkvg_surface_destroy(VkvgSurface surf)
 
 #if VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
        vkDestroySemaphore (surf->dev->vkDev, surf->timeline, NULL);
+#else
+       vkDestroyFence (surf->dev->vkDev, surf->flushFence, NULL);
 #endif
 
        vkvg_device_destroy (surf->dev);
@@ -308,9 +336,7 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){
                                                                                 VMA_MEMORY_USAGE_GPU_ONLY,
                                                                                 VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT);
 
-       VkCommandBuffer cmd = dev->cmd;
-       _device_wait_and_reset_device_fence (dev);
-
+       VkCommandBuffer cmd = surf->cmd;
        vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
        vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT,
                                                  VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
@@ -330,7 +356,8 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){
                                         vkh_image_get_vkimage (stagImg),  VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST);
 
        vkh_cmd_end             (cmd);
-       _device_submit_cmd              (dev, &cmd, dev->fence);
+
+       _surface_submit_cmd (surf);
 
        VkhImage stagImgLinear = stagImg;
 
@@ -345,7 +372,6 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){
                        .dstOffset = {0},
                        .extent = {(int32_t)surf->width, (int32_t)surf->height, 1}
                };
-               _device_wait_and_reset_device_fence (dev);
 
                vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
                vkh_image_set_layout (cmd, stagImgLinear, VK_IMAGE_ASPECT_COLOR_BIT,
@@ -359,12 +385,11 @@ vkvg_status_t vkvg_surface_write_to_png (VkvgSurface surf, const char* path){
                                           vkh_image_get_vkimage (stagImgLinear),  VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
 
                vkh_cmd_end             (cmd);
-               _device_submit_cmd              (dev, &cmd, dev->fence);
 
-               vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
+               _surface_submit_cmd (surf);
+
                vkh_image_destroy (stagImg);
-       } else
-               vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
+       }
 
        void* img = vkh_image_map (stagImgLinear);
 
@@ -399,8 +424,7 @@ vkvg_status_t vkvg_surface_write_to_memory (VkvgSurface surf, unsigned char* con
                                                                                 VMA_MEMORY_USAGE_GPU_TO_CPU,
                                                                                 VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT);
 
-       VkCommandBuffer cmd = dev->cmd;
-       _device_wait_and_reset_device_fence (dev);
+       VkCommandBuffer cmd = surf->cmd;
 
        vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
        vkh_image_set_layout (cmd, stagImg, VK_IMAGE_ASPECT_COLOR_BIT,
@@ -421,8 +445,8 @@ vkvg_status_t vkvg_surface_write_to_memory (VkvgSurface surf, unsigned char* con
                                         vkh_image_get_vkimage (stagImg),  VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST);
 
        vkh_cmd_end             (cmd);
-       _device_submit_cmd              (dev, &cmd, dev->fence);
-       vkWaitForFences (dev->vkDev, 1, &dev->fence, VK_TRUE, UINT64_MAX);
+
+       _surface_submit_cmd (surf);
 
        uint64_t stride = vkh_image_get_stride(stagImg);
        uint32_t dest_stride = surf->width * 4;

diff --git a/src/vkvg_surface_internal.c b/src/vkvg_surface_internal.c
index 8c575c3b0cf824d085c386b6311de7445552934f..4ef66686fb1c42eeb95a8664b7afb1977f620c85 100644 (file)
--- a/src/vkvg_surface_internal.c
+++ b/src/vkvg_surface_internal.c
@@ -20,15 +20,13 @@
  * THE SOFTWARE.
  */
 
-#include "vkvg_surface_internal.h"
 #include "vkvg_device_internal.h"
+#include "vkvg_surface_internal.h"
 #include "vkh_image.h"
+#include "vkh_queue.h"
 
 void _explicit_ms_resolve (VkvgSurface surf){
-       VkvgDevice              dev = surf->dev;
-       VkCommandBuffer cmd = dev->cmd;
-
-       _device_wait_and_reset_device_fence (dev);
+       VkCommandBuffer cmd = surf->cmd;
 
        vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
        vkh_image_set_layout (cmd, surf->imgMS, VK_IMAGE_ASPECT_COLOR_BIT,
@@ -53,15 +51,12 @@ void _explicit_ms_resolve (VkvgSurface surf){
                                                  VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
        vkh_cmd_end (cmd);
 
-       _device_submit_cmd (dev, &cmd, dev->fence);
+       _surface_submit_cmd (surf);
 }
 
 void _clear_surface (VkvgSurface surf, VkImageAspectFlags aspect)
 {
-       VkvgDevice              dev = surf->dev;
-       VkCommandBuffer cmd = dev->cmd;
-
-       _device_wait_and_reset_device_fence (dev);
+       VkCommandBuffer cmd = surf->cmd;
 
        vkh_cmd_begin (cmd, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
 
@@ -101,7 +96,7 @@ void _clear_surface (VkvgSurface surf, VkImageAspectFlags aspect)
        }
        vkh_cmd_end (cmd);
 
-       _device_submit_cmd (dev, &cmd, dev->fence);
+       _surface_submit_cmd (surf);
 }
 
 void _create_surface_main_image (VkvgSurface surf){
@@ -186,9 +181,51 @@ VkvgSurface _create_surface (VkvgDevice dev, VkFormat format) {
        surf->format = format;
        if (dev->threadAware)
                mtx_init (&surf->mutex, mtx_plain);
+       surf->cmdPool = vkh_cmd_pool_create ((VkhDevice)dev, dev->gQueue->familyIndex, VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT);
+       vkh_cmd_buffs_create((VkhDevice)dev, surf->cmdPool,VK_COMMAND_BUFFER_LEVEL_PRIMARY, 1, &surf->cmd);
 
 #if VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
        surf->timeline = vkh_timeline_create ((VkhDevice)dev, 0);
+#else
+       surf->flushFence = vkh_fence_create ((VkhDevice)dev);
+#endif
+
+#if defined(DEBUG) && defined(VKVG_DBG_UTILS)
+       vkh_device_set_object_name((VkhDevice)dev, VK_OBJECT_TYPE_COMMAND_BUFFER, (uint64_t)surf->cmd, "vkvgSurfCmd");
 #endif
+
        return surf;
 }
+//if fence sync, surf mutex must be locked.
+/*bool _surface_wait_cmd (VkvgSurface surf) {
+       LOG(VKVG_LOG_INFO, "SURF: _surface__wait_flush_fence\n");
+#ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
+       if (vkh_timeline_wait ((VkhDevice)surf->dev, surf->timeline, surf->timelineStep) == VK_SUCCESS)
+               return true;
+#else
+       if (WaitForFences (surf->dev->vkDev, 1, &surf->flushFence, VK_TRUE, VKVG_FENCE_TIMEOUT) == VK_SUCCESS) {
+               ResetFences (surf->dev->vkDev, 1, &surf->flushFence);
+               return true;
+       }
+#endif
+       LOG(VKVG_LOG_DEBUG, "CTX: _wait_flush_fence timeout\n");
+       surf->status = VKVG_STATUS_TIMEOUT;
+       return false;
+}*/
+//surface mutex must be locked to call this method, locking to guard also the surf->cmd local buffer usage.
+void _surface_submit_cmd (VkvgSurface surf) {
+       VkvgDevice dev = surf->dev;
+#ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
+       LOCK_DEVICE
+       vkh_cmd_submit_timelined (dev->gQueue, &surf->cmd, surf->timeline, surf->timelineStep, surf->timelineStep+1);
+       surf->timelineStep++;
+       UNLOCK_DEVICE
+       vkh_timeline_wait ((VkhDevice)dev, surf->timeline, surf->timelineStep);
+#else
+       LOCK_DEVICE
+       vkh_cmd_submit (surf->dev->gQueue, &surf->cmd, surf->flushFence);
+       UNLOCK_DEVICE
+       WaitForFences (surf->dev->vkDev, 1, &surf->flushFence, VK_TRUE, VKVG_FENCE_TIMEOUT);
+       ResetFences (surf->dev->vkDev, 1, &surf->flushFence);
+#endif
+}

diff --git a/src/vkvg_surface_internal.h b/src/vkvg_surface_internal.h
index 0b52d4bf3e719b60f5aeb393658b22cc5208f429..5cc47c670445d324940c5ad61f8574a3dee09a9a 100644 (file)
--- a/src/vkvg_surface_internal.h
+++ b/src/vkvg_surface_internal.h
@@ -37,12 +37,17 @@ typedef struct _vkvg_surface_t {
        VkhImage                img;
        VkhImage                imgMS;
        VkhImage                stencil;
+       VkCommandPool   cmdPool;                                //local pools ensure thread safety
+       VkCommandBuffer cmd;                                    //surface local command buffer.
        bool                    newSurf;
        mtx_t                   mutex;
 #ifdef VKVG_ENABLE_VK_TIMELINE_SEMAPHORE
        VkSemaphore             timeline;                               /**< Timeline semaphore */
        uint64_t                timelineStep;
+#else
+       VkFence                 flushFence;                             //unsignaled idle.
 #endif
+
 }vkvg_surface;
 
 #define LOCK_SURFACE(surf) \
@@ -59,4 +64,7 @@ void _create_surface_secondary_images (VkvgSurface surf);
 void _create_framebuffer (VkvgSurface surf);
 void _create_surface_images (VkvgSurface surf);
 VkvgSurface _create_surface (VkvgDevice dev, VkFormat format);
+
+void _surface_submit_cmd (VkvgSurface surf);
+//bool _surface_wait_cmd (VkvgSurface surf);
 #endif