Google Git
Sign in
android / platform / external / mesa3d / b74eb12a8e01ac086ecfa4f6448a3e46b2cb1593 / . / src / virtio / vulkan / vn_feedback.c
blob: d85340455d28af437e3f7e6b80a6cea5634f8344 [file] [log] [blame]
/*
* Copyright 2022 Google LLC
* SPDX-License-Identifier: MIT
*/
#include "vn_feedback.h"
#include "vn_command_buffer.h"
#include "vn_device.h"
#include "vn_physical_device.h"
#include "vn_query_pool.h"
#include "vn_queue.h"
static uint32_t
vn_get_memory_type_index(const VkPhysicalDeviceMemoryProperties *mem_props,
uint32_t mem_type_bits,
VkMemoryPropertyFlags required_mem_flags)
{
u_foreach_bit(mem_type_index, mem_type_bits)
{
assert(mem_type_index < mem_props->memoryTypeCount);
if ((mem_props->memoryTypes[mem_type_index].propertyFlags &
required_mem_flags) == required_mem_flags)
return mem_type_index;
}
return UINT32_MAX;
}
VkResult
vn_feedback_buffer_create(struct vn_device *dev,
uint32_t size,
const VkAllocationCallbacks *alloc,
struct vn_feedback_buffer **out_fb_buf)
{
const bool exclusive = dev->queue_family_count == 1;
const VkPhysicalDeviceMemoryProperties *mem_props =
&dev->physical_device->memory_properties;
VkDevice dev_handle = vn_device_to_handle(dev);
VkResult result;
struct vn_feedback_buffer *fb_buf =
vk_zalloc(alloc, sizeof(*fb_buf), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!fb_buf)
return VK_ERROR_OUT_OF_HOST_MEMORY;
/* use concurrent to avoid explicit queue family ownership transfer for
* device created with queues from multiple queue families
*/
const VkBufferCreateInfo buf_create_info = {
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.size = size,
/* Feedback for fences and timeline semaphores will write to this buffer
* as a DST when signalling. Timeline semaphore feedback will also read
* from this buffer as a SRC to retrieve the counter value to signal.
*/
.usage =
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode =
exclusive ? VK_SHARING_MODE_EXCLUSIVE : VK_SHARING_MODE_CONCURRENT,
/* below favors the current venus protocol */
.queueFamilyIndexCount = exclusive ? 0 : dev->queue_family_count,
.pQueueFamilyIndices = exclusive ? NULL : dev->queue_families,
};
result = vn_CreateBuffer(dev_handle, &buf_create_info, alloc,
&fb_buf->buf_handle);
if (result != VK_SUCCESS)
goto out_free_feedback_buffer;
struct vn_buffer *buf = vn_buffer_from_handle(fb_buf->buf_handle);
const VkMemoryRequirements *mem_req =
&buf->requirements.memory.memoryRequirements;
const uint32_t mem_type_index =
vn_get_memory_type_index(mem_props, mem_req->memoryTypeBits,
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (mem_type_index >= mem_props->memoryTypeCount) {
result = VK_ERROR_INITIALIZATION_FAILED;
goto out_destroy_buffer;
}
const VkMemoryAllocateInfo mem_alloc_info = {
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.allocationSize = mem_req->size,
.memoryTypeIndex = mem_type_index,
};
result = vn_AllocateMemory(dev_handle, &mem_alloc_info, alloc,
&fb_buf->mem_handle);
if (result != VK_SUCCESS)
goto out_destroy_buffer;
const VkBindBufferMemoryInfo bind_info = {
.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
.buffer = fb_buf->buf_handle,
.memory = fb_buf->mem_handle,
.memoryOffset = 0,
};
result = vn_BindBufferMemory2(dev_handle, 1, &bind_info);
if (result != VK_SUCCESS)
goto out_free_memory;
result = vn_MapMemory(dev_handle, fb_buf->mem_handle, 0, VK_WHOLE_SIZE, 0,
&fb_buf->data);
if (result != VK_SUCCESS)
goto out_free_memory;
*out_fb_buf = fb_buf;
return VK_SUCCESS;
out_free_memory:
vn_FreeMemory(dev_handle, fb_buf->mem_handle, alloc);
out_destroy_buffer:
vn_DestroyBuffer(dev_handle, fb_buf->buf_handle, alloc);
out_free_feedback_buffer:
vk_free(alloc, fb_buf);
return result;
}
void
vn_feedback_buffer_destroy(struct vn_device *dev,
struct vn_feedback_buffer *fb_buf,
const VkAllocationCallbacks *alloc)
{
VkDevice dev_handle = vn_device_to_handle(dev);
vn_UnmapMemory(dev_handle, fb_buf->mem_handle);
vn_FreeMemory(dev_handle, fb_buf->mem_handle, alloc);
vn_DestroyBuffer(dev_handle, fb_buf->buf_handle, alloc);
vk_free(alloc, fb_buf);
}
static inline uint32_t
vn_get_feedback_buffer_alignment(struct vn_device *dev,
struct vn_feedback_buffer *fb_buf)
{
struct vn_buffer *buf = vn_buffer_from_handle(fb_buf->buf_handle);
return align(buf->requirements.memory.memoryRequirements.alignment,
dev->physical_device->wa_min_fb_align);
}
static VkResult
vn_feedback_pool_grow_locked(struct vn_feedback_pool *pool)
{
VN_TRACE_FUNC();
struct vn_feedback_buffer *fb_buf = NULL;
VkResult result;
result =
vn_feedback_buffer_create(pool->dev, pool->size, pool->alloc, &fb_buf);
if (result != VK_SUCCESS)
return result;
pool->used = 0;
pool->alignment = vn_get_feedback_buffer_alignment(pool->dev, fb_buf);
list_add(&fb_buf->head, &pool->fb_bufs);
return VK_SUCCESS;
}
VkResult
vn_feedback_pool_init(struct vn_device *dev,
struct vn_feedback_pool *pool,
uint32_t size,
const VkAllocationCallbacks *alloc)
{
simple_mtx_init(&pool->mutex, mtx_plain);
pool->dev = dev;
pool->alloc = alloc;
pool->size = size;
pool->used = size;
pool->alignment = 1;
list_inithead(&pool->fb_bufs);
list_inithead(&pool->free_slots);
return VK_SUCCESS;
}
void
vn_feedback_pool_fini(struct vn_feedback_pool *pool)
{
list_for_each_entry_safe(struct vn_feedback_slot, slot, &pool->free_slots,
head)
vk_free(pool->alloc, slot);
list_for_each_entry_safe(struct vn_feedback_buffer, fb_buf, &pool->fb_bufs,
head)
vn_feedback_buffer_destroy(pool->dev, fb_buf, pool->alloc);
simple_mtx_destroy(&pool->mutex);
}
static struct vn_feedback_buffer *
vn_feedback_pool_alloc_locked(struct vn_feedback_pool *pool,
uint32_t size,
uint32_t *out_offset)
{
/* Default values of pool->used and pool->alignment are used to trigger the
* initial pool grow, and will be properly initialized after that.
*/
if (unlikely(align(size, pool->alignment) > pool->size - pool->used)) {
VkResult result = vn_feedback_pool_grow_locked(pool);
if (result != VK_SUCCESS)
return NULL;
assert(align(size, pool->alignment) <= pool->size - pool->used);
}
*out_offset = pool->used;
pool->used += align(size, pool->alignment);
return list_first_entry(&pool->fb_bufs, struct vn_feedback_buffer, head);
}
struct vn_feedback_slot *
vn_feedback_pool_alloc(struct vn_feedback_pool *pool,
enum vn_feedback_type type)
{
static const uint32_t slot_size = 8;
struct vn_feedback_buffer *fb_buf;
uint32_t offset;
struct vn_feedback_slot *slot;
simple_mtx_lock(&pool->mutex);
if (!list_is_empty(&pool->free_slots)) {
slot =
list_first_entry(&pool->free_slots, struct vn_feedback_slot, head);
list_del(&slot->head);
simple_mtx_unlock(&pool->mutex);
slot->type = type;
return slot;
}
slot = vk_alloc(pool->alloc, sizeof(*slot), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!slot) {
simple_mtx_unlock(&pool->mutex);
return NULL;
}
fb_buf = vn_feedback_pool_alloc_locked(pool, slot_size, &offset);
simple_mtx_unlock(&pool->mutex);
if (!fb_buf) {
vk_free(pool->alloc, slot);
return NULL;
}
slot->type = type;
slot->offset = offset;
slot->buf_handle = fb_buf->buf_handle;
slot->data = fb_buf->data + offset;
return slot;
}
void
vn_feedback_pool_free(struct vn_feedback_pool *pool,
struct vn_feedback_slot *slot)
{
simple_mtx_lock(&pool->mutex);
list_add(&slot->head, &pool->free_slots);
simple_mtx_unlock(&pool->mutex);
}
static inline bool
mask_is_32bit(uint64_t x)
{
return (x & 0xffffffff00000000) == 0;
}
static void
vn_build_buffer_memory_barrier(const VkDependencyInfo *dep_info,
VkBufferMemoryBarrier *barrier1,
VkPipelineStageFlags *src_stage_mask,
VkPipelineStageFlags *dst_stage_mask)
{
assert(dep_info->pNext == NULL);
assert(dep_info->memoryBarrierCount == 0);
assert(dep_info->bufferMemoryBarrierCount == 1);
assert(dep_info->imageMemoryBarrierCount == 0);
const VkBufferMemoryBarrier2 *barrier2 =
&dep_info->pBufferMemoryBarriers[0];
assert(barrier2->pNext == NULL);
assert(mask_is_32bit(barrier2->srcStageMask));
assert(mask_is_32bit(barrier2->srcAccessMask));
assert(mask_is_32bit(barrier2->dstStageMask));
assert(mask_is_32bit(barrier2->dstAccessMask));
*barrier1 = (VkBufferMemoryBarrier){
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = barrier2->srcAccessMask,
.dstAccessMask = barrier2->dstAccessMask,
.srcQueueFamilyIndex = barrier2->srcQueueFamilyIndex,
.dstQueueFamilyIndex = barrier2->dstQueueFamilyIndex,
.buffer = barrier2->buffer,
.offset = barrier2->offset,
.size = barrier2->size,
};
*src_stage_mask = barrier2->srcStageMask;
*dst_stage_mask = barrier2->dstStageMask;
}
static void
vn_cmd_buffer_memory_barrier(VkCommandBuffer cmd_handle,
const VkDependencyInfo *dep_info,
bool sync2)
{
if (sync2)
vn_CmdPipelineBarrier2(cmd_handle, dep_info);
else {
VkBufferMemoryBarrier barrier1;
VkPipelineStageFlags src_stage_mask;
VkPipelineStageFlags dst_stage_mask;
vn_build_buffer_memory_barrier(dep_info, &barrier1, &src_stage_mask,
&dst_stage_mask);
vn_CmdPipelineBarrier(cmd_handle, src_stage_mask, dst_stage_mask,
dep_info->dependencyFlags, 0, NULL, 1, &barrier1,
0, NULL);
}
}
void
vn_event_feedback_cmd_record(VkCommandBuffer cmd_handle,
VkEvent ev_handle,
VkPipelineStageFlags2 src_stage_mask,
VkResult status,
bool sync2)
{
/* For vkCmdSetEvent and vkCmdResetEvent feedback interception.
*
* The injection point is after the event call to avoid introducing
* unexpected src stage waiting for VK_PIPELINE_STAGE_HOST_BIT and
* VK_PIPELINE_STAGE_TRANSFER_BIT if they are not already being waited by
* vkCmdSetEvent or vkCmdResetEvent. On the other hand, the delay in the
* feedback signal is acceptable for the nature of VkEvent, and the event
* feedback cmds lifecycle is guarded by the intercepted command buffer.
*/
struct vn_event *ev = vn_event_from_handle(ev_handle);
struct vn_feedback_slot *slot = ev->feedback_slot;
if (!slot)
return;
STATIC_ASSERT(sizeof(*slot->status) == 4);
const VkDependencyInfo dep_before = {
.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
.dependencyFlags = 0,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers =
(VkBufferMemoryBarrier2[]){
{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2,
.srcStageMask = src_stage_mask | VK_PIPELINE_STAGE_HOST_BIT |
VK_PIPELINE_STAGE_TRANSFER_BIT,
.srcAccessMask =
VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = slot->buf_handle,
.offset = slot->offset,
.size = 4,
},
},
};
vn_cmd_buffer_memory_barrier(cmd_handle, &dep_before, sync2);
vn_CmdFillBuffer(cmd_handle, slot->buf_handle, slot->offset, 4, status);
const VkDependencyInfo dep_after = {
.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
.dependencyFlags = 0,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers =
(VkBufferMemoryBarrier2[]){
{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_HOST_BIT,
.dstAccessMask =
VK_ACCESS_HOST_READ_BIT | VK_ACCESS_HOST_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = slot->buf_handle,
.offset = slot->offset,
.size = 4,
},
},
};
vn_cmd_buffer_memory_barrier(cmd_handle, &dep_after, sync2);
}
static inline void
vn_feedback_cmd_record_flush_barrier(VkCommandBuffer cmd_handle,
VkBuffer buffer,
VkDeviceSize offset,
VkDeviceSize size)
{
const VkBufferMemoryBarrier buf_flush_barrier = {
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_HOST_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = buffer,
.offset = offset,
.size = size,
};
vn_CmdPipelineBarrier(cmd_handle, VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_HOST_BIT, 0, 0, NULL, 1,
&buf_flush_barrier, 0, NULL);
}
static VkResult
vn_feedback_cmd_record(VkCommandBuffer cmd_handle,
struct vn_feedback_slot *dst_slot,
struct vn_feedback_slot *src_slot)
{
STATIC_ASSERT(sizeof(*dst_slot->status) == 4);
STATIC_ASSERT(sizeof(*dst_slot->counter) == 8);
STATIC_ASSERT(sizeof(*src_slot->counter) == 8);
/* slot size is 8 bytes for timeline semaphore and 4 bytes fence.
* src slot is non-null for timeline semaphore.
*/
const VkDeviceSize buf_size = src_slot ? 8 : 4;
static const VkCommandBufferBeginInfo begin_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.pNext = NULL,
.flags = 0,
.pInheritanceInfo = NULL,
};
VkResult result = vn_BeginCommandBuffer(cmd_handle, &begin_info);
if (result != VK_SUCCESS)
return result;
static const VkMemoryBarrier mem_barrier_before = {
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
.pNext = NULL,
/* make pending writes available to stay close to signal op */
.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
/* no need to make all memory visible for feedback update */
.dstAccessMask = 0,
};
const VkBufferMemoryBarrier buf_barrier_before = {
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = NULL,
/* slot memory has been made available via mem_barrier_before */
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = dst_slot->buf_handle,
.offset = dst_slot->offset,
.size = buf_size,
};
/* host writes for src_slots should implicitly be made visible upon
* QueueSubmit call */
vn_CmdPipelineBarrier(cmd_handle, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1,
&mem_barrier_before, 1, &buf_barrier_before, 0,
NULL);
/* If passed a src_slot, timeline semaphore feedback records a
* cmd to copy the counter value from the src slot to the dst slot.
* If src_slot is NULL, then fence feedback records a cmd to fill
* the dst slot with VK_SUCCESS.
*/
if (src_slot) {
assert(src_slot->type == VN_FEEDBACK_TYPE_SEMAPHORE);
assert(dst_slot->type == VN_FEEDBACK_TYPE_SEMAPHORE);
const VkBufferCopy buffer_copy = {
.srcOffset = src_slot->offset,
.dstOffset = dst_slot->offset,
.size = buf_size,
};
vn_CmdCopyBuffer(cmd_handle, src_slot->buf_handle, dst_slot->buf_handle,
1, &buffer_copy);
} else {
assert(dst_slot->type == VN_FEEDBACK_TYPE_FENCE);
vn_CmdFillBuffer(cmd_handle, dst_slot->buf_handle, dst_slot->offset,
buf_size, VK_SUCCESS);
}
vn_feedback_cmd_record_flush_barrier(cmd_handle, dst_slot->buf_handle,
dst_slot->offset, buf_size);
return vn_EndCommandBuffer(cmd_handle);
}
struct vn_semaphore_feedback_cmd *
vn_semaphore_feedback_cmd_alloc(struct vn_device *dev,
struct vn_feedback_slot *dst_slot)
{
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
struct vn_semaphore_feedback_cmd *sfb_cmd;
VkCommandBuffer *cmd_handles;
VK_MULTIALLOC(ma);
vk_multialloc_add(&ma, &sfb_cmd, __typeof__(*sfb_cmd), 1);
vk_multialloc_add(&ma, &cmd_handles, __typeof__(*cmd_handles),
dev->queue_family_count);
if (!vk_multialloc_zalloc(&ma, alloc, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT))
return NULL;
struct vn_feedback_slot *src_slot =
vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE);
if (!src_slot) {
vk_free(alloc, sfb_cmd);
return NULL;
}
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
VkDevice dev_handle = vn_device_to_handle(dev);
VkResult result =
vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], dst_slot,
src_slot, &cmd_handles[i]);
if (result != VK_SUCCESS) {
for (uint32_t j = 0; j < i; j++) {
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j],
cmd_handles[j]);
}
vn_feedback_pool_free(&dev->feedback_pool, src_slot);
vk_free(alloc, sfb_cmd);
return NULL;
}
}
sfb_cmd->cmd_handles = cmd_handles;
sfb_cmd->src_slot = src_slot;
return sfb_cmd;
}
void
vn_semaphore_feedback_cmd_free(struct vn_device *dev,
struct vn_semaphore_feedback_cmd *sfb_cmd)
{
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
vn_feedback_cmd_free(vn_device_to_handle(dev), &dev->fb_cmd_pools[i],
sfb_cmd->cmd_handles[i]);
}
vn_feedback_pool_free(&dev->feedback_pool, sfb_cmd->src_slot);
vk_free(alloc, sfb_cmd);
}
static void
vn_query_feedback_cmd_record_internal(VkCommandBuffer cmd_handle,
VkQueryPool pool_handle,
uint32_t query,
uint32_t count,
bool copy)
{
struct vn_query_pool *pool = vn_query_pool_from_handle(pool_handle);
assert(pool->fb_buf);
/* Results are always 64 bit and include availability bit (also 64 bit) */
const VkDeviceSize slot_size = (pool->result_array_size * 8) + 8;
const VkDeviceSize offset = slot_size * query;
const VkDeviceSize buf_size = slot_size * count;
/* The first synchronization scope of vkCmdCopyQueryPoolResults does not
* include the query feedback buffer. Insert a barrier to ensure ordering
* against feedback buffer fill cmd injected in vkCmdResetQueryPool.
*
* The second synchronization scope of vkCmdResetQueryPool does not include
* the query feedback buffer. Insert a barrer to ensure ordering against
* prior cmds referencing the queries.
*
* For srcAccessMask, VK_ACCESS_TRANSFER_WRITE_BIT is sufficient since the
* gpu cache invalidation for feedback buffer fill in vkResetQueryPool is
* done implicitly via queue submission.
*/
const VkPipelineStageFlags src_stage_mask =
copy ? VK_PIPELINE_STAGE_TRANSFER_BIT
: VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
const VkBufferMemoryBarrier buf_barrier_before = {
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = pool->fb_buf->buf_handle,
.offset = offset,
.size = buf_size,
};
vn_CmdPipelineBarrier(cmd_handle, src_stage_mask,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 1,
&buf_barrier_before, 0, NULL);
if (copy) {
/* Per spec: "The first synchronization scope includes all commands
* which reference the queries in queryPool indicated by query that
* occur earlier in submission order. If flags does not include
* VK_QUERY_RESULT_WAIT_BIT, vkCmdEndQueryIndexedEXT,
* vkCmdWriteTimestamp2, vkCmdEndQuery, and vkCmdWriteTimestamp are
* excluded from this scope."
*
* Set VK_QUERY_RESULT_WAIT_BIT to ensure ordering after
* vkCmdEndQuery or vkCmdWriteTimestamp makes the query available.
*
* Set VK_QUERY_RESULT_64_BIT as we can convert it to 32 bit if app
* requested that.
*
* Per spec: "vkCmdCopyQueryPoolResults is considered to be a transfer
* operation, and its writes to buffer memory must be synchronized using
* VK_PIPELINE_STAGE_TRANSFER_BIT and VK_ACCESS_TRANSFER_WRITE_BIT
* before using the results."
*
* So we can reuse the flush barrier after this copy cmd.
*/
vn_CmdCopyQueryPoolResults(cmd_handle, pool_handle, query, count,
pool->fb_buf->buf_handle, offset, slot_size,
VK_QUERY_RESULT_WITH_AVAILABILITY_BIT |
VK_QUERY_RESULT_64_BIT |
VK_QUERY_RESULT_WAIT_BIT);
} else {
vn_CmdFillBuffer(cmd_handle, pool->fb_buf->buf_handle, offset, buf_size,
0);
}
vn_feedback_cmd_record_flush_barrier(cmd_handle, pool->fb_buf->buf_handle,
offset, buf_size);
}
static VkResult
vn_query_feedback_cmd_record(VkDevice dev_handle,
struct list_head *query_records,
struct vn_query_feedback_cmd *qfb_cmd)
{
assert(!list_is_empty(query_records));
static const VkCommandBufferBeginInfo begin_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
};
VkResult result = vn_BeginCommandBuffer(qfb_cmd->cmd_handle, &begin_info);
if (result != VK_SUCCESS)
return result;
list_for_each_entry_safe(struct vn_cmd_query_record, record, query_records,
head) {
vn_query_feedback_cmd_record_internal(
qfb_cmd->cmd_handle, vn_query_pool_to_handle(record->query_pool),
record->query, record->query_count, record->copy);
}
return vn_EndCommandBuffer(qfb_cmd->cmd_handle);
}
VkResult
vn_query_feedback_cmd_alloc(VkDevice dev_handle,
struct vn_feedback_cmd_pool *fb_cmd_pool,
struct list_head *query_records,
struct vn_query_feedback_cmd **out_qfb_cmd)
{
struct vn_query_feedback_cmd *qfb_cmd;
VkResult result;
simple_mtx_lock(&fb_cmd_pool->mutex);
if (list_is_empty(&fb_cmd_pool->free_qfb_cmds)) {
struct vn_command_pool *cmd_pool =
vn_command_pool_from_handle(fb_cmd_pool->pool_handle);
qfb_cmd = vk_alloc(&cmd_pool->allocator, sizeof(*qfb_cmd),
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!qfb_cmd) {
result = VK_ERROR_OUT_OF_HOST_MEMORY;
goto out_unlock;
}
const VkCommandBufferAllocateInfo info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
.commandPool = fb_cmd_pool->pool_handle,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
.commandBufferCount = 1,
};
VkCommandBuffer qfb_cmd_handle;
result = vn_AllocateCommandBuffers(dev_handle, &info, &qfb_cmd_handle);
if (result != VK_SUCCESS) {
vk_free(&cmd_pool->allocator, qfb_cmd);
goto out_unlock;
}
qfb_cmd->fb_cmd_pool = fb_cmd_pool;
qfb_cmd->cmd_handle = qfb_cmd_handle;
} else {
qfb_cmd = list_first_entry(&fb_cmd_pool->free_qfb_cmds,
struct vn_query_feedback_cmd, head);
list_del(&qfb_cmd->head);
vn_ResetCommandBuffer(qfb_cmd->cmd_handle, 0);
}
result = vn_query_feedback_cmd_record(dev_handle, query_records, qfb_cmd);
if (result != VK_SUCCESS) {
list_add(&qfb_cmd->head, &fb_cmd_pool->free_qfb_cmds);
goto out_unlock;
}
*out_qfb_cmd = qfb_cmd;
out_unlock:
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
void
vn_query_feedback_cmd_free(struct vn_query_feedback_cmd *qfb_cmd)
{
simple_mtx_lock(&qfb_cmd->fb_cmd_pool->mutex);
list_add(&qfb_cmd->head, &qfb_cmd->fb_cmd_pool->free_qfb_cmds);
simple_mtx_unlock(&qfb_cmd->fb_cmd_pool->mutex);
}
VkResult
vn_feedback_cmd_alloc(VkDevice dev_handle,
struct vn_feedback_cmd_pool *fb_cmd_pool,
struct vn_feedback_slot *dst_slot,
struct vn_feedback_slot *src_slot,
VkCommandBuffer *out_cmd_handle)
{
VkCommandPool cmd_pool_handle = fb_cmd_pool->pool_handle;
const VkCommandBufferAllocateInfo info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
.pNext = NULL,
.commandPool = cmd_pool_handle,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
.commandBufferCount = 1,
};
VkCommandBuffer cmd_handle;
VkResult result;
simple_mtx_lock(&fb_cmd_pool->mutex);
result = vn_AllocateCommandBuffers(dev_handle, &info, &cmd_handle);
if (result != VK_SUCCESS)
goto out_unlock;
result = vn_feedback_cmd_record(cmd_handle, dst_slot, src_slot);
if (result != VK_SUCCESS) {
vn_FreeCommandBuffers(dev_handle, cmd_pool_handle, 1, &cmd_handle);
goto out_unlock;
}
*out_cmd_handle = cmd_handle;
out_unlock:
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
void
vn_feedback_cmd_free(VkDevice dev_handle,
struct vn_feedback_cmd_pool *fb_cmd_pool,
VkCommandBuffer cmd_handle)
{
simple_mtx_lock(&fb_cmd_pool->mutex);
vn_FreeCommandBuffers(dev_handle, fb_cmd_pool->pool_handle, 1,
&cmd_handle);
simple_mtx_unlock(&fb_cmd_pool->mutex);
}
VkResult
vn_feedback_cmd_pools_init(struct vn_device *dev)
{
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
VkDevice dev_handle = vn_device_to_handle(dev);
struct vn_feedback_cmd_pool *fb_cmd_pools;
VkCommandPoolCreateInfo info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.pNext = NULL,
.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
};
if (VN_PERF(NO_FENCE_FEEDBACK) && VN_PERF(NO_SEMAPHORE_FEEDBACK) &&
VN_PERF(NO_QUERY_FEEDBACK))
return VK_SUCCESS;
assert(dev->queue_family_count);
fb_cmd_pools =
vk_zalloc(alloc, sizeof(*fb_cmd_pools) * dev->queue_family_count,
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
if (!fb_cmd_pools)
return VK_ERROR_OUT_OF_HOST_MEMORY;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
VkResult result;
info.queueFamilyIndex = dev->queue_families[i];
result = vn_CreateCommandPool(dev_handle, &info, alloc,
&fb_cmd_pools[i].pool_handle);
if (result != VK_SUCCESS) {
for (uint32_t j = 0; j < i; j++) {
vn_DestroyCommandPool(dev_handle, fb_cmd_pools[j].pool_handle,
alloc);
simple_mtx_destroy(&fb_cmd_pools[j].mutex);
}
vk_free(alloc, fb_cmd_pools);
return result;
}
simple_mtx_init(&fb_cmd_pools[i].mutex, mtx_plain);
list_inithead(&fb_cmd_pools[i].free_qfb_cmds);
}
dev->fb_cmd_pools = fb_cmd_pools;
return VK_SUCCESS;
}
void
vn_feedback_cmd_pools_fini(struct vn_device *dev)
{
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
VkDevice dev_handle = vn_device_to_handle(dev);
if (!dev->fb_cmd_pools)
return;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
list_for_each_entry_safe(struct vn_query_feedback_cmd, feedback_cmd,
&dev->fb_cmd_pools[i].free_qfb_cmds, head)
vk_free(alloc, feedback_cmd);
vn_DestroyCommandPool(dev_handle, dev->fb_cmd_pools[i].pool_handle,
alloc);
simple_mtx_destroy(&dev->fb_cmd_pools[i].mutex);
}
vk_free(alloc, dev->fb_cmd_pools);
}