Merge "Refactor AndroidConsoleAgents to GraphicsAgents."
diff --git a/host-common/address_space_graphics.cpp b/host-common/address_space_graphics.cpp
index 8c03bb4..36914b9 100644
--- a/host-common/address_space_graphics.cpp
+++ b/host-common/address_space_graphics.cpp
@@ -51,6 +51,7 @@
     bool fromLoad;
     uint64_t size;
     uint64_t hostmemId;
+    void *externalAddr;
 };
 
 struct Block {
@@ -63,13 +64,14 @@
     size_t dedicatedSize = 0;
     bool usesVirtioGpuHostmem = false;
     uint64_t hostmemId = 0;
+    bool external = false;
 };
 
 class Globals {
 public:
     Globals() :
         mPerContextBufferSize(
-                android_hw->hw_gltransport_asg_writeBufferSize) { }
+                aemu_get_android_hw()->hw_gltransport_asg_writeBufferSize) { }
 
     ~Globals() { clear(); }
 
@@ -415,16 +417,23 @@
     void fillBlockLocked(Block& block, struct AllocationCreateInfo& create) {
         if (create.dedicated) {
             if (create.virtioGpu) {
-                void* buf = aligned_buf_alloc(ADDRESS_SPACE_GRAPHICS_PAGE_SIZE, create.size);
+                void* buf;
 
-                struct MemEntry entry = { 0 };
-                entry.hva = (uint64_t)(uintptr_t)buf;
-                entry.size = create.size;
-                entry.register_fixed = create.hostmemRegisterFixed;
-                entry.fixed_id = create.hostmemId ? create.hostmemId : 0;
-                entry.caching = MAP_CACHE_CACHED;
+                if (create.externalAddr) {
+                    buf = create.externalAddr;
+                    block.external = true;
+                } else {
+                    buf = aligned_buf_alloc(ADDRESS_SPACE_GRAPHICS_PAGE_SIZE, create.size);
 
-                create.hostmemId = mControlOps->hostmem_register(&entry);
+                    struct MemEntry entry = { 0 };
+                    entry.hva = (uint64_t)(uintptr_t)buf;
+                    entry.size = create.size;
+                    entry.register_fixed = create.hostmemRegisterFixed;
+                    entry.fixed_id = create.hostmemId ? create.hostmemId : 0;
+                    entry.caching = MAP_CACHE_CACHED;
+
+                    create.hostmemId = mControlOps->hostmem_register(&entry);
+                }
 
                 block.buffer = (char*)buf;
                 block.subAlloc =
@@ -493,9 +502,9 @@
 
     void destroyBlockLocked(Block& block) {
 
-        if (block.usesVirtioGpuHostmem) {
+        if (block.usesVirtioGpuHostmem && !block.external) {
             mControlOps->hostmem_unregister(block.hostmemId);
-        } else {
+        } else if (!block.external) {
             mControlOps->remove_memory_mapping(
                 get_address_space_device_hw_funcs()->getPhysAddrStartLocked() +
                     block.offsetIntoPhys,
@@ -507,8 +516,9 @@
         }
 
         delete block.subAlloc;
-
-        aligned_buf_free(block.buffer);
+        if (!block.external) {
+            aligned_buf_free(block.buffer);
+        }
 
         block.isEmpty = true;
     }
@@ -588,7 +598,7 @@
     mHostContext.ring_config->buffer_size =
         sGlobals()->perContextBufferSize();
     mHostContext.ring_config->flush_interval =
-        android_hw->hw_gltransport_asg_writeStepSize;
+        aemu_get_android_hw()->hw_gltransport_asg_writeStepSize;
     mHostContext.ring_config->host_consumed_pos = 0;
     mHostContext.ring_config->guest_write_pos = 0;
     mHostContext.ring_config->transfer_mode = 1;
@@ -743,7 +753,7 @@
     mHostContext.ring_config->buffer_size =
         sGlobals()->perContextBufferSize();
     mHostContext.ring_config->flush_interval =
-        android_hw->hw_gltransport_asg_writeStepSize;
+        aemu_get_android_hw()->hw_gltransport_asg_writeStepSize;
 
     // In load, the live ring config state is in shared host/guest ram.
     //
diff --git a/host-common/address_space_graphics_unittests.cpp b/host-common/address_space_graphics_unittests.cpp
index 16d66c0..9370d91 100644
--- a/host-common/address_space_graphics_unittests.cpp
+++ b/host-common/address_space_graphics_unittests.cpp
@@ -86,9 +86,9 @@
             auto setVersionResult = ping((uint64_t)ASG_SET_VERSION, mVersion);
             uint32_t hostVersion = setVersionResult.size;
             EXPECT_LE(hostVersion, mVersion);
-            EXPECT_EQ(android_hw->hw_gltransport_asg_writeStepSize,
+            EXPECT_EQ(aemu_get_android_hw()->hw_gltransport_asg_writeStepSize,
                       mContext.ring_config->flush_interval);
-            EXPECT_EQ(android_hw->hw_gltransport_asg_writeBufferSize,
+            EXPECT_EQ(aemu_get_android_hw()->hw_gltransport_asg_writeBufferSize,
                       mBufferSize);
 
             mContext.ring_config->transfer_mode = 1;
@@ -560,8 +560,8 @@
     static void TearDownTestSuite() { }
 
     void SetUp() override {
-        android_hw->hw_gltransport_asg_writeBufferSize = 524288;
-        android_hw->hw_gltransport_asg_writeStepSize = 1024;
+        aemu_get_android_hw()->hw_gltransport_asg_writeBufferSize = 524288;
+        aemu_get_android_hw()->hw_gltransport_asg_writeStepSize = 1024;
 
         mDevice = HostAddressSpaceDevice::get();
         ConsumerInterface interface = {
@@ -600,8 +600,8 @@
     void TearDown() override {
         AddressSpaceGraphicsContext::clear();
         mDevice->clear();
-        android_hw->hw_gltransport_asg_writeBufferSize = 524288;
-        android_hw->hw_gltransport_asg_writeStepSize = 1024;
+        aemu_get_android_hw()->hw_gltransport_asg_writeBufferSize = 524288;
+        aemu_get_android_hw()->hw_gltransport_asg_writeStepSize = 1024;
         EXPECT_EQ(nullptr, mCurrentConsumer);
     }
 
@@ -625,8 +625,8 @@
             std::vector<char> expectedRead(trip.readBytes, ASG_TEST_READ_PATTERN);
             std::vector<char> toRead(trip.readBytes, 0);
 
-            size_t stepSize = android_hw->hw_gltransport_asg_writeStepSize;
-            size_t stepSizeRead = android_hw->hw_gltransport_asg_writeBufferSize;
+            size_t stepSize = aemu_get_android_hw()->hw_gltransport_asg_writeStepSize;
+            size_t stepSizeRead = aemu_get_android_hw()->hw_gltransport_asg_writeBufferSize;
 
             size_t sent = 0;
             while (sent < trip.writeBytes) {
@@ -672,7 +672,7 @@
 // Tests writing via an IOStream-like interface
 // (allocBuffer, then flush)
 TEST_F(AddressSpaceGraphicsTest, BasicWrite) {
-    EXPECT_EQ(1024, android_hw->hw_gltransport_asg_writeStepSize);
+    EXPECT_EQ(1024, aemu_get_android_hw()->hw_gltransport_asg_writeStepSize);
     Client client(mDevice);
 
     // Tests that going over the step size results in nullptr
@@ -688,7 +688,7 @@
 
 // Tests that further allocs result in flushing
 TEST_F(AddressSpaceGraphicsTest, FlushFromAlloc) {
-    EXPECT_EQ(1024, android_hw->hw_gltransport_asg_writeStepSize);
+    EXPECT_EQ(1024, aemu_get_android_hw()->hw_gltransport_asg_writeStepSize);
     Client client(mDevice);
 
     auto buf = client.allocBuffer(1024);
diff --git a/host-common/include/host-common/FeatureControlDefGuest.h b/host-common/include/host-common/FeatureControlDefGuest.h
index f808e31..0136a1e 100644
--- a/host-common/include/host-common/FeatureControlDefGuest.h
+++ b/host-common/include/host-common/FeatureControlDefGuest.h
@@ -55,11 +55,21 @@
 FEATURE_CONTROL_ITEM(VulkanIgnoredHandles)
 FEATURE_CONTROL_ITEM(VirtioGpuNext)
 FEATURE_CONTROL_ITEM(Mac80211hwsimUserspaceManaged)
-FEATURE_CONTROL_ITEM(HasSharedSlotsHostMemoryAllocator)
 FEATURE_CONTROL_ITEM(HardwareDecoder)
 FEATURE_CONTROL_ITEM(VirtioWifi)
 FEATURE_CONTROL_ITEM(ModemSimulator)
 FEATURE_CONTROL_ITEM(VirtioMouse)
 FEATURE_CONTROL_ITEM(VirtconsoleLogcat)
+FEATURE_CONTROL_ITEM(VirtioVsockPipe)
 FEATURE_CONTROL_ITEM(VulkanQueueSubmitWithCommands)
 FEATURE_CONTROL_ITEM(VulkanBatchedDescriptorSetUpdate)
+FEATURE_CONTROL_ITEM(Minigbm)
+FEATURE_CONTROL_ITEM(GnssGrpcV1)
+FEATURE_CONTROL_ITEM(AndroidbootProps)
+FEATURE_CONTROL_ITEM(AndroidbootProps2)
+FEATURE_CONTROL_ITEM(DeviceSkinOverlay)
+FEATURE_CONTROL_ITEM(BluetoothEmulation)
+FEATURE_CONTROL_ITEM(DeviceStateOnBoot)
+FEATURE_CONTROL_ITEM(HWCMultiConfigs)
+FEATURE_CONTROL_ITEM(VirtioSndCard)
+FEATURE_CONTROL_ITEM(VirtioTablet)
diff --git a/host-common/include/host-common/FeatureControlDefHost.h b/host-common/include/host-common/FeatureControlDefHost.h
index d769e03..72cbae9 100644
--- a/host-common/include/host-common/FeatureControlDefHost.h
+++ b/host-common/include/host-common/FeatureControlDefHost.h
@@ -55,11 +55,11 @@
 FEATURE_CONTROL_ITEM(Vulkan)
 FEATURE_CONTROL_ITEM(MacroUi)
 FEATURE_CONTROL_ITEM(IpDisconnectOnLoad)
+FEATURE_CONTROL_ITEM(HasSharedSlotsHostMemoryAllocator)
 FEATURE_CONTROL_ITEM(CarVHalTable)
 FEATURE_CONTROL_ITEM(VulkanSnapshots)
 FEATURE_CONTROL_ITEM(DynamicMediaProfile)
 FEATURE_CONTROL_ITEM(CarVhalReplay)
-FEATURE_CONTROL_ITEM(CarAssistButton)
 FEATURE_CONTROL_ITEM(NoDelayCloseColorBuffer)
 FEATURE_CONTROL_ITEM(NoDeviceFrame)
 FEATURE_CONTROL_ITEM(VirtioGpuNativeSync)
@@ -69,9 +69,10 @@
 FEATURE_CONTROL_ITEM(NativeTextureDecompression)
 FEATURE_CONTROL_ITEM(BptcTextureSupport)
 FEATURE_CONTROL_ITEM(GuestUsesAngle)
-FEATURE_CONTROL_ITEM(VirtioVsockPipe)
 FEATURE_CONTROL_ITEM(S3tcTextureSupport)
 FEATURE_CONTROL_ITEM(RgtcTextureSupport)
 FEATURE_CONTROL_ITEM(VulkanNativeSwapchain)
 FEATURE_CONTROL_ITEM(VirtioGpuFenceContexts)
 FEATURE_CONTROL_ITEM(AsyncComposeSupport)
+FEATURE_CONTROL_ITEM(NoDraw)
+FEATURE_CONTROL_ITEM(MigratableSnapshotSave)
diff --git a/host-common/include/host-common/globals.h b/host-common/include/host-common/globals.h
index 301b528..2631e27 100644
--- a/host-common/include/host-common/globals.h
+++ b/host-common/include/host-common/globals.h
@@ -34,9 +34,6 @@
 /* MSVC only exports function pointers */
 extern AEMU_EXPORT AvdInfo** aemu_get_android_avdInfoPtr();
 
-/* the hardware configuration for this specific virtual device */
-extern AndroidHwConfig   android_hw[1];
-
 /* MSVC only exports function pointers */
 AEMU_EXPORT AndroidHwConfig* aemu_get_android_hw();
 
diff --git a/host-common/opengl/emugl_config.cpp b/host-common/opengl/emugl_config.cpp
index 35d228d..18f52b3 100644
--- a/host-common/opengl/emugl_config.cpp
+++ b/host-common/opengl/emugl_config.cpp
@@ -155,7 +155,7 @@
 }
 
 bool emuglConfig_current_renderer_supports_snapshot() {
-    if (android_hw->hw_arc) {
+    if (aemu_get_android_hw()->hw_arc) {
         return sCurrentRenderer == SELECTED_RENDERER_OFF ||
                sCurrentRenderer == SELECTED_RENDERER_GUEST;
     }
diff --git a/stream-servers/FrameBuffer.cpp b/stream-servers/FrameBuffer.cpp
index 8d4ed4b..896bf05 100644
--- a/stream-servers/FrameBuffer.cpp
+++ b/stream-servers/FrameBuffer.cpp
@@ -1135,8 +1135,19 @@
                         return false;
                     }
                     INFO("Recreating swapchain...");
-                    m_displayVk->bindToSurface(m_vkSurface, static_cast<uint32_t>(m_windowWidth),
-                                             static_cast<uint32_t>(m_windowHeight));
+                    int maxRetries = 8;
+                    while (maxRetries>=0 && !m_displayVk->bindToSurface(
+                                                       m_vkSurface,
+                                                       static_cast<uint32_t>(m_windowWidth),
+                                                       static_cast<uint32_t>(m_windowHeight))) {
+                        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                        --maxRetries;
+                        INFO("Swapchain recreation failed, retrying...");
+                    }
+                    if (maxRetries < 0) {
+                        GFXSTREAM_ABORT(FatalError(ABORT_REASON_OTHER))
+                            << "Failed to create Swapchain.";
+                    }
                     INFO("Recreating swapchain completes.");
                     return true;
                 }
diff --git a/stream-servers/FrameBuffer.h b/stream-servers/FrameBuffer.h
index 41b46b8..3a32398 100644
--- a/stream-servers/FrameBuffer.h
+++ b/stream-servers/FrameBuffer.h
@@ -638,8 +638,8 @@
     int m_y = 0;
     int m_framebufferWidth = 0;
     int m_framebufferHeight = 0;
-    int m_windowWidth = 0;
-    int m_windowHeight = 0;
+    std::atomic_int m_windowWidth = 0;
+    std::atomic_int m_windowHeight = 0;
     float m_dpr = 0;
 
     bool m_useSubWindow = false;
diff --git a/stream-servers/tests/SwapChainStateVk_unittest.cpp b/stream-servers/tests/SwapChainStateVk_unittest.cpp
index 819dd22..dd4d21d 100644
--- a/stream-servers/tests/SwapChainStateVk_unittest.cpp
+++ b/stream-servers/tests/SwapChainStateVk_unittest.cpp
@@ -149,5 +149,6 @@
         *k_vk, m_vkSurface, m_vkPhysicalDevice, k_width, k_height,
         {m_swapChainQueueFamilyIndex});
     ASSERT_NE(swapChainCi, std::nullopt);
-    SwapChainStateVk swapChainState(*k_vk, m_vkDevice, swapChainCi->mCreateInfo);
+    std::unique_ptr<SwapChainStateVk> swapChainState =
+        SwapChainStateVk::createSwapChainVk(*k_vk, m_vkDevice, swapChainCi->mCreateInfo);
 }
\ No newline at end of file
diff --git a/stream-servers/vulkan/DisplayVk.cpp b/stream-servers/vulkan/DisplayVk.cpp
index 5c240b1..619ed30 100644
--- a/stream-servers/vulkan/DisplayVk.cpp
+++ b/stream-servers/vulkan/DisplayVk.cpp
@@ -88,7 +88,7 @@
     m_vk.vkDestroyCommandPool(m_vkDevice, m_vkCommandPool, nullptr);
 }
 
-void DisplayVk::bindToSurface(VkSurfaceKHR surface, uint32_t width, uint32_t height) {
+bool DisplayVk::bindToSurface(VkSurfaceKHR surface, uint32_t width, uint32_t height) {
     {
         android::base::AutoLock lock(*m_compositorVkQueueLock);
         VK_CHECK(vk_util::waitForVkQueueIdleWithRetry(m_vk, m_compositorVkQueue));
@@ -109,8 +109,7 @@
         m_vk, surface, m_vkPhysicalDevice, width, height,
         {m_swapChainQueueFamilyIndex, m_compositorQueueFamilyIndex});
     if (!swapChainCi) {
-        GFXSTREAM_ABORT(FatalError(ABORT_REASON_OTHER))
-            << "Failed to create VkSwapchainCreateInfoKHR.";
+        return false;
     }
     VkFormatProperties formatProps;
     m_vk.vkGetPhysicalDeviceFormatProperties(m_vkPhysicalDevice,
@@ -121,7 +120,8 @@
                "attachment, and therefore can't be used as the render target of CompositorVk.";
     }
     m_swapChainStateVk =
-        std::make_unique<SwapChainStateVk>(m_vk, m_vkDevice, swapChainCi->mCreateInfo);
+        SwapChainStateVk::createSwapChainVk(m_vk, m_vkDevice, swapChainCi->mCreateInfo);
+    if (m_swapChainStateVk == nullptr) return false;
 
     int numSwapChainImages = m_swapChainStateVk->getVkImages().size();
 
@@ -136,6 +136,7 @@
     surfaceState->m_height = height;
     surfaceState->m_width = width;
     m_surfaceState = std::move(surfaceState);
+    return true;
 }
 
 std::tuple<bool, std::shared_future<void>> DisplayVk::post(
diff --git a/stream-servers/vulkan/DisplayVk.h b/stream-servers/vulkan/DisplayVk.h
index 89cfa5b..835cd5f 100644
--- a/stream-servers/vulkan/DisplayVk.h
+++ b/stream-servers/vulkan/DisplayVk.h
@@ -28,7 +28,7 @@
               VkQueue compositorVkQueue, std::shared_ptr<android::base::Lock> compositorVkQueueLock,
               VkQueue swapChainVkQueue, std::shared_ptr<android::base::Lock> swapChainVkQueueLock);
     ~DisplayVk();
-    void bindToSurface(VkSurfaceKHR, uint32_t width, uint32_t height);
+    bool bindToSurface(VkSurfaceKHR, uint32_t width, uint32_t height);
     // The first component of the returned tuple is false when the swapchain is no longer valid and
     // bindToSurface() needs to be called again. When the first component is true, the second
     // component of the returned tuple is a/ future that will complete when the GPU side of work
diff --git a/stream-servers/vulkan/SwapChainStateVk.cpp b/stream-servers/vulkan/SwapChainStateVk.cpp
index 83e9749..22d3857 100644
--- a/stream-servers/vulkan/SwapChainStateVk.cpp
+++ b/stream-servers/vulkan/SwapChainStateVk.cpp
@@ -72,14 +72,30 @@
     }
 }
 
-SwapChainStateVk::SwapChainStateVk(const goldfish_vk::VulkanDispatch& vk, VkDevice vkDevice,
-                                   const VkSwapchainCreateInfoKHR& swapChainCi)
+
+
+std::unique_ptr<SwapChainStateVk> SwapChainStateVk::createSwapChainVk(
+    const goldfish_vk::VulkanDispatch& vk,
+    VkDevice vkDevice, const VkSwapchainCreateInfoKHR& swapChainCi) {
+    std::unique_ptr<SwapChainStateVk> swapChainVk(new SwapChainStateVk(vk, vkDevice));
+    if (swapChainVk->initSwapChainStateVk(swapChainCi) != VK_SUCCESS) {
+        return nullptr;
+    }
+    return swapChainVk;
+}
+
+SwapChainStateVk::SwapChainStateVk(const goldfish_vk::VulkanDispatch& vk, VkDevice vkDevice)
     : m_vk(vk),
       m_vkDevice(vkDevice),
       m_vkSwapChain(VK_NULL_HANDLE),
       m_vkImages(0),
       m_vkImageViews(0) {
-    VK_CHECK(m_vk.vkCreateSwapchainKHR(m_vkDevice, &swapChainCi, nullptr, &m_vkSwapChain));
+}
+
+VkResult SwapChainStateVk::initSwapChainStateVk(const VkSwapchainCreateInfoKHR& swapChainCi) {
+    VkResult res = m_vk.vkCreateSwapchainKHR(m_vkDevice, &swapChainCi, nullptr, &m_vkSwapChain);
+    if (res == VK_ERROR_INITIALIZATION_FAILED) return res;
+    VK_CHECK(res);
     uint32_t imageCount = 0;
     VK_CHECK(m_vk.vkGetSwapchainImagesKHR(m_vkDevice, m_vkSwapChain, &imageCount, nullptr));
     m_vkImages.resize(imageCount);
@@ -104,13 +120,16 @@
         VK_CHECK(m_vk.vkCreateImageView(m_vkDevice, &imageViewCi, nullptr, &vkImageView));
         m_vkImageViews.push_back(vkImageView);
     }
+    return VK_SUCCESS;
 }
 
 SwapChainStateVk::~SwapChainStateVk() {
     for (auto imageView : m_vkImageViews) {
         m_vk.vkDestroyImageView(m_vkDevice, imageView, nullptr);
     }
-    m_vk.vkDestroySwapchainKHR(m_vkDevice, m_vkSwapChain, nullptr);
+    if (m_vkSwapChain != VK_NULL_HANDLE) {
+        m_vk.vkDestroySwapchainKHR(m_vkDevice, m_vkSwapChain, nullptr);
+    }
 }
 
 std::vector<const char*> SwapChainStateVk::getRequiredInstanceExtensions() {
diff --git a/stream-servers/vulkan/SwapChainStateVk.h b/stream-servers/vulkan/SwapChainStateVk.h
index f9c900f..5da74d4 100644
--- a/stream-servers/vulkan/SwapChainStateVk.h
+++ b/stream-servers/vulkan/SwapChainStateVk.h
@@ -40,8 +40,13 @@
         const goldfish_vk::VulkanDispatch&, VkSurfaceKHR, VkPhysicalDevice, uint32_t width,
         uint32_t height, const std::unordered_set<uint32_t>& queueFamilyIndices);
 
-    explicit SwapChainStateVk(const goldfish_vk::VulkanDispatch&, VkDevice,
-                              const VkSwapchainCreateInfoKHR&);
+    SwapChainStateVk() = delete;
+    SwapChainStateVk(const SwapChainStateVk&) = delete;
+    SwapChainStateVk& operator = (const SwapChainStateVk&) = delete;
+
+    static std::unique_ptr<SwapChainStateVk> createSwapChainVk(const goldfish_vk::VulkanDispatch&,
+                                                        VkDevice, const VkSwapchainCreateInfoKHR&);
+
     ~SwapChainStateVk();
     VkFormat getFormat();
     const std::vector<VkImage>& getVkImages() const;
@@ -49,6 +54,9 @@
     VkSwapchainKHR getSwapChain() const;
 
    private:
+    explicit SwapChainStateVk(const goldfish_vk::VulkanDispatch&, VkDevice);
+
+    VkResult initSwapChainStateVk(const VkSwapchainCreateInfoKHR& swapChainCi);
     const static VkFormat k_vkFormat = VK_FORMAT_B8G8R8A8_UNORM;
     const static VkColorSpaceKHR k_vkColorSpace = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR;
 
diff --git a/stream-servers/vulkan/VkDecoderGlobalState.cpp b/stream-servers/vulkan/VkDecoderGlobalState.cpp
index e0cb05b..19599a4 100644
--- a/stream-servers/vulkan/VkDecoderGlobalState.cpp
+++ b/stream-servers/vulkan/VkDecoderGlobalState.cpp
@@ -1143,7 +1143,7 @@
             fprintf(stderr, "%s: track the new device (begin)\n", __func__);
         }
 
-        if(!swiftshader) {
+        if (!swiftshader) {
             lock = std::make_unique<AutoLock>(mLock);
         }
 
@@ -1168,8 +1168,8 @@
         }
 
         init_vulkan_dispatch_from_device(vk, *pDevice, dispatch_VkDevice(boxed));
-        deviceInfo.externalFencePool = std::make_unique<ExternalFencePool<VulkanDispatch>>(
-            dispatch_VkDevice(boxed), *pDevice);
+        deviceInfo.externalFencePool =
+            std::make_unique<ExternalFencePool<VulkanDispatch>>(dispatch_VkDevice(boxed), *pDevice);
 
         if (mLogging) {
             fprintf(stderr, "%s: init vulkan dispatch from device (end)\n", __func__);
@@ -1786,8 +1786,7 @@
         {
             AutoLock lock(mLock);
             for (uint32_t i = 0; i < fenceCount; i++) {
-                if (pFences[i] == VK_NULL_HANDLE)
-                    continue;
+                if (pFences[i] == VK_NULL_HANDLE) continue;
 
                 DCHECK(mFenceInfo.find(pFences[i]) != mFenceInfo.end());
                 if (mFenceInfo[pFences[i]].external) {
@@ -1805,12 +1804,12 @@
         // For external fences, we unilaterally put them in the pool to ensure they finish
         // TODO: should store creation info / pNext chain per fence and re-apply?
         VkFenceCreateInfo createInfo{
-            .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, .pNext = 0, .flags = 0 };
+            .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, .pNext = 0, .flags = 0};
         auto deviceInfo = mDeviceInfo.find(device);
         if (deviceInfo == mDeviceInfo.end()) {
             return VK_ERROR_OUT_OF_DEVICE_MEMORY;
         }
-        for (auto fence: externalFences) {
+        for (auto fence : externalFences) {
             VkFence replacement = deviceInfo->second.externalFencePool->pop(&createInfo);
             if (replacement == VK_NULL_HANDLE) {
                 VK_CHECK(vk->vkCreateFence(device, &createInfo, 0, &replacement));
@@ -2547,7 +2546,9 @@
         if (cmdBufferInfoIt == mCmdBufferInfo.end()) {
             return;
         }
-        auto deviceInfoIt = mDeviceInfo.find(cmdBufferInfoIt->second.device);
+        const auto& cmdBufferInfo = cmdBufferInfoIt->second;
+
+        auto deviceInfoIt = mDeviceInfo.find(cmdBufferInfo.device);
         if (deviceInfoIt == mDeviceInfo.end()) {
             return;
         }
@@ -2601,8 +2602,7 @@
                         srcBarrier.oldLayout, srcBarrier.newLayout);
             }
 
-            VkResult result =
-                it->second.cmpInfo.initDecomp(vk, cmdBufferInfoIt->second.device, image);
+            VkResult result = it->second.cmpInfo.initDecomp(vk, cmdBufferInfo.device, image);
             if (result != VK_SUCCESS) {
                 fprintf(stderr, "WARNING: texture decompression failed\n");
                 continue;
@@ -2660,16 +2660,16 @@
                                      currImageBarriers.data()   // pImageMemoryBarriers
             );
         }
-        if (needRebind && cmdBufferInfoIt->second.computePipeline) {
+        if (needRebind && cmdBufferInfo.computePipeline) {
             // Recover pipeline bindings
             vk->vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE,
-                                  cmdBufferInfoIt->second.computePipeline);
-            if (cmdBufferInfoIt->second.descriptorSets.size() > 0) {
+                                  cmdBufferInfo.computePipeline);
+            if (cmdBufferInfo.descriptorSets.size() > 0) {
                 vk->vkCmdBindDescriptorSets(
-                    commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE,
-                    cmdBufferInfoIt->second.descriptorLayout, cmdBufferInfoIt->second.firstSet,
-                    cmdBufferInfoIt->second.descriptorSets.size(),
-                    cmdBufferInfoIt->second.descriptorSets.data(), 0, nullptr);
+                    commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, cmdBufferInfo.descriptorLayout,
+                    cmdBufferInfo.firstSet, cmdBufferInfo.descriptorSets.size(),
+                    cmdBufferInfo.descriptorSets.data(), cmdBufferInfo.dynamicOffsets.size(),
+                    cmdBufferInfo.dynamicOffsets.data());
             }
         }
         if (memoryBarrierCount || bufferMemoryBarrierCount || !persistentImageBarriers.empty()) {
@@ -3528,12 +3528,14 @@
         VkResult result = vk->vkResetCommandBuffer(commandBuffer, flags);
         if (VK_SUCCESS == result) {
             AutoLock lock(mLock);
-            mCmdBufferInfo[commandBuffer].preprocessFuncs.clear();
-            mCmdBufferInfo[commandBuffer].subCmds.clear();
-            mCmdBufferInfo[commandBuffer].computePipeline = 0;
-            mCmdBufferInfo[commandBuffer].firstSet = 0;
-            mCmdBufferInfo[commandBuffer].descriptorLayout = 0;
-            mCmdBufferInfo[commandBuffer].descriptorSets.clear();
+            auto& bufferInfo = mCmdBufferInfo[commandBuffer];
+            bufferInfo.preprocessFuncs.clear();
+            bufferInfo.subCmds.clear();
+            bufferInfo.computePipeline = VK_NULL_HANDLE;
+            bufferInfo.firstSet = 0;
+            bufferInfo.descriptorLayout = VK_NULL_HANDLE;
+            bufferInfo.descriptorSets.clear();
+            bufferInfo.dynamicOffsets.clear();
         }
         return result;
     }
@@ -3888,11 +3890,15 @@
             AutoLock lock(mLock);
             auto cmdBufferInfoIt = mCmdBufferInfo.find(commandBuffer);
             if (cmdBufferInfoIt != mCmdBufferInfo.end()) {
-                cmdBufferInfoIt->second.descriptorLayout = layout;
+                auto& cmdBufferInfo = cmdBufferInfoIt->second;
+                cmdBufferInfo.descriptorLayout = layout;
+
                 if (descriptorSetCount) {
-                    cmdBufferInfoIt->second.firstSet = firstSet;
-                    cmdBufferInfoIt->second.descriptorSets.assign(
-                        pDescriptorSets, pDescriptorSets + descriptorSetCount);
+                    cmdBufferInfo.firstSet = firstSet;
+                    cmdBufferInfo.descriptorSets.assign(pDescriptorSets,
+                                                        pDescriptorSets + descriptorSetCount);
+                    cmdBufferInfo.dynamicOffsets.assign(pDynamicOffsets,
+                                                        pDynamicOffsets + dynamicOffsetCount);
                 }
             }
         }
@@ -5803,6 +5809,7 @@
         uint32_t firstSet = 0;
         VkPipelineLayout descriptorLayout = 0;
         std::vector<VkDescriptorSet> descriptorSets;
+        std::vector<uint32_t> dynamicOffsets;
         uint32_t sequenceNumber = 0;
     };
 
@@ -7423,9 +7430,9 @@
 template <class TDispatch>
 ExternalFencePool<TDispatch>::~ExternalFencePool() {
     if (!mPool.empty()) {
-        GFXSTREAM_ABORT(FatalError(ABORT_REASON_OTHER)) <<
-            "External fence pool for device " << static_cast<void*>(mDevice) << " destroyed but " <<
-            mPool.size() << " fences still not destroyed.";
+        GFXSTREAM_ABORT(FatalError(ABORT_REASON_OTHER))
+            << "External fence pool for device " << static_cast<void*>(mDevice) << " destroyed but "
+            << mPool.size() << " fences still not destroyed.";
     }
 }
 
@@ -7444,19 +7451,18 @@
     VkFence fence = VK_NULL_HANDLE;
     {
         AutoLock lock(mLock);
-        auto it = std::find_if(mPool.begin(), mPool.end(),
-                               [this](const VkFence& fence) {
-                                   VkResult status = m_vk->vkGetFenceStatus(mDevice, fence);
-                                   if (status != VK_SUCCESS) {
-                                       if (status != VK_NOT_READY) {
-                                           VK_CHECK(status);
-                                       }
+        auto it = std::find_if(mPool.begin(), mPool.end(), [this](const VkFence& fence) {
+            VkResult status = m_vk->vkGetFenceStatus(mDevice, fence);
+            if (status != VK_SUCCESS) {
+                if (status != VK_NOT_READY) {
+                    VK_CHECK(status);
+                }
 
-                                       // Status is valid, but fence is not yet signaled
-                                       return false;
-                                   }
-                                   return true;
-                               });
+                // Status is valid, but fence is not yet signaled
+                return false;
+            }
+            return true;
+        });
         if (it == mPool.end()) {
             return VK_NULL_HANDLE;
         }