vulkan/gles interop: ensure consistent selection of physdevs / interop This should solve two potential issues: 1. Use VkPhysicalDeviceIDProperties to advertise only the physical device we selected to the guest. If we ended up selecting a different physical device in VkCommonOperations versus on_vkCreateDevice from the guest, problems can happen. 2. Ensure device UUID between gles/vk match. If they don't, we can't use interop, even if the gl side reports support for the interop extensions (think NV gl alongside Intel vk). The gles side should have a way to query the UUID of the device (tested with native GL and GLES on Linux and native GL on Windows); otherwise, we can't use zero-copy interop. Bug: 190561902 Change-Id: I6c6e217a3c09566ab4c0d47a91432c58343efc54
diff --git a/stream-servers/FrameBuffer.cpp b/stream-servers/FrameBuffer.cpp index b9966e0..daef6c0 100644 --- a/stream-servers/FrameBuffer.cpp +++ b/stream-servers/FrameBuffer.cpp
@@ -354,20 +354,29 @@ // used by underlying EGL driver might become invalid, // preventing new contexts from being created that share // against those contexts. + goldfish_vk::VkEmulation* vkEmu = nullptr; if (feature_is_enabled(kFeature_Vulkan)) { auto dispatch = emugl::vkDispatch(false /* not for testing */); - auto emu = goldfish_vk::createOrGetGlobalVkEmulation(dispatch); + vkEmu = goldfish_vk::createOrGetGlobalVkEmulation(dispatch); bool useDeferredCommands = android::base::getEnvironmentVariable("ANDROID_EMU_VK_DISABLE_DEFERRED_COMMANDS").empty(); bool useCreateResourcesWithRequirements = android::base::getEnvironmentVariable("ANDROID_EMU_VK_DISABLE_USE_CREATE_RESOURCES_WITH_REQUIREMENTS").empty(); - goldfish_vk::setUseDeferredCommands(emu, useDeferredCommands); - goldfish_vk::setUseCreateResourcesWithRequirements(emu, useCreateResourcesWithRequirements); + goldfish_vk::setUseDeferredCommands(vkEmu, useDeferredCommands); + goldfish_vk::setUseCreateResourcesWithRequirements(vkEmu, useCreateResourcesWithRequirements); if (feature_is_enabled(kFeature_VulkanNativeSwapchain)) { fb->m_displayVk = std::make_unique<DisplayVk>( - *dispatch, emu->physdev, emu->queueFamilyIndex, - emu->queueFamilyIndex, emu->device, emu->queue, emu->queue); - fb->m_vkInstance = emu->instance; + *dispatch, vkEmu->physdev, vkEmu->queueFamilyIndex, + vkEmu->queueFamilyIndex, vkEmu->device, vkEmu->queue, vkEmu->queue); + fb->m_vkInstance = vkEmu->instance; + } + if (vkEmu->deviceInfo.supportsIdProperties) { + GL_LOG("Supports id properties, got a vulkan device UUID"); + fprintf(stderr, "%s: Supports id properties, got a vulkan device UUID\n", __func__); + memcpy(fb->m_vulkanUUID, vkEmu->deviceInfo.idProps.deviceUUID, VK_UUID_SIZE); + } else { + GL_LOG("Doesn't support id properties, no vulkan device UUID"); + fprintf(stderr, "%s: Doesn't support id properties, no vulkan device UUID\n", __func__); } } @@ -624,6 +633,47 @@ fb->m_glRenderer = std::string((const char*)s_gles2.glGetString(GL_RENDERER)); fb->m_glVersion = std::string((const char*)s_gles2.glGetString(GL_VERSION)); + // Attempt to get the device UUID of the gles and match with Vulkan. If + // they match, interop is possible. If they don't, then don't trust the + // result of interop query to egl and fall back to CPU copy, as we might + // have initialized Vulkan devices and GLES contexts from different + // physical devices. + + bool vkglesUuidsGood = true; + + // First, if the VkEmulation instance doesn't support ext memory capabilities, + // it won't support uuids. + if (!vkEmu || !vkEmu->deviceInfo.supportsIdProperties) { + vkglesUuidsGood = false; + } + + s_gles2.glGetError(); + + GLint numDeviceUuids = 0; + s_gles2.glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &numDeviceUuids); + + // If underlying gles doesn't support UUID query, we definitely don't + // support interop and should not proceed further. + + if (!numDeviceUuids || 1 != numDeviceUuids) { + // If numDeviceUuids != 1 it's unclear what gles we're using (SLI? Xinerama?) + // and we shouldn't try to interop. + vkglesUuidsGood = false; + } + + if (vkglesUuidsGood && 1 == numDeviceUuids) { + s_gles2.glGetUnsignedBytei_vEXT(GL_DEVICE_UUID_EXT, 0, fb->m_glesUUID); + GL_LOG("Underlying gles supports UUID"); + if (0 == memcmp(fb->m_vulkanUUID, fb->m_glesUUID, VK_UUID_SIZE)) { + GL_LOG("vk/gles UUIDs match"); + fprintf(stderr, "%s: vk/gles UUIDs match\n", __func__); + } else { + GL_LOG("vk/gles UUIDs do not match"); + fprintf(stderr, "%s: vk/gles UUIDs do not match\n", __func__); + vkglesUuidsGood = false; + } + } + DBG("GL Vendor %s\n", fb->m_glVendor.c_str()); DBG("GL Renderer %s\n", fb->m_glRenderer.c_str()); DBG("GL Extensions %s\n", fb->m_glVersion.c_str()); @@ -641,6 +691,9 @@ if (s_egl.eglQueryVulkanInteropSupportANDROID) { fb->m_vulkanInteropSupported = s_egl.eglQueryVulkanInteropSupportANDROID(); + if (!vkglesUuidsGood) { + fb->m_vulkanInteropSupported = false; + } } fprintf(stderr, "%s: interop? %d\n", __func__, fb->m_vulkanInteropSupported); @@ -768,6 +821,8 @@ setDisplayPose(displayId, 0, 0, getWidth(), getHeight(), 0); m_perfThread->start(); + memset(m_vulkanUUID, 0x0, 16); + memset(m_glesUUID, 0x0, 16); } FrameBuffer::~FrameBuffer() {
diff --git a/stream-servers/FrameBuffer.h b/stream-servers/FrameBuffer.h index 42ff034..79353ba 100644 --- a/stream-servers/FrameBuffer.h +++ b/stream-servers/FrameBuffer.h
@@ -780,5 +780,10 @@ std::unique_ptr<DisplayVk> m_displayVk; VkInstance m_vkInstance = VK_NULL_HANDLE; VkSurfaceKHR m_vkSurface = VK_NULL_HANDLE; + + // UUIDs of physical devices for Vulkan and GLES, respectively. In most + // cases, this determines whether we can support zero-copy interop. + uint8_t m_vulkanUUID[VK_UUID_SIZE]; + uint8_t m_glesUUID[VK_UUID_SIZE]; }; #endif
diff --git a/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp b/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp index b31fa34..a8cfd56 100644 --- a/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp +++ b/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp
@@ -4476,11 +4476,13 @@ // Common between GL_EXT_memory_object and GL_EXT_semaphore GL_APICALL void GL_APIENTRY glGetUnsignedBytevEXT(GLenum pname, GLubyte* data) { GET_CTX_V2(); + if (!ctx->dispatcher().glGetUnsignedBytevEXT) return; ctx->dispatcher().glGetUnsignedBytevEXT(pname, data); } GL_APICALL void GL_APIENTRY glGetUnsignedBytei_vEXT(GLenum target, GLuint index, GLubyte* data) { GET_CTX_V2(); + if (!ctx->dispatcher().glGetUnsignedBytei_vEXT) return; ctx->dispatcher().glGetUnsignedBytei_vEXT(target, index, data); }
diff --git a/stream-servers/vulkan/VkCommonOperations.cpp b/stream-servers/vulkan/VkCommonOperations.cpp index b0b37bf..18bda70 100644 --- a/stream-servers/vulkan/VkCommonOperations.cpp +++ b/stream-servers/vulkan/VkCommonOperations.cpp
@@ -602,6 +602,18 @@ ivk->vkGetInstanceProcAddr( sVkEmulation->instance, "vkGetPhysicalDeviceImageFormatProperties2KHR")); + sVkEmulation->getPhysicalDeviceProperties2Func = reinterpret_cast< + PFN_vkGetPhysicalDeviceProperties2KHR>( + ivk->vkGetInstanceProcAddr( + sVkEmulation->instance, + "vkGetPhysicalDeviceProperties2KHR")); + if (!sVkEmulation->getPhysicalDeviceProperties2Func) { + sVkEmulation->getPhysicalDeviceProperties2Func = reinterpret_cast< + PFN_vkGetPhysicalDeviceProperties2KHR>( + ivk->vkGetInstanceProcAddr( + sVkEmulation->instance, + "vkGetPhysicalDeviceProperties2")); + } } if (sVkEmulation->instanceSupportsMoltenVK) { @@ -664,6 +676,25 @@ if (sVkEmulation->instanceSupportsExternalMemoryCapabilities) { deviceInfos[i].supportsExternalMemory = extensionsSupported( deviceExts, externalMemoryDeviceExtNames); + deviceInfos[i].supportsIdProperties = + sVkEmulation->getPhysicalDeviceProperties2Func != nullptr; + if (!sVkEmulation->getPhysicalDeviceProperties2Func) { + fprintf(stderr, "%s: warning: device should support ID properties " + "but vkGetPhysicalDeviceProperties2 could not be found\n", __func__); + } + } + + if (deviceInfos[i].supportsIdProperties) { + VkPhysicalDeviceIDPropertiesKHR idProps = { + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR, nullptr, + }; + VkPhysicalDeviceProperties2KHR propsWithId = { + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR, &idProps, + }; + sVkEmulation->getPhysicalDeviceProperties2Func( + physdevs[i], + &propsWithId); + deviceInfos[i].idProps = idProps; } uint32_t queueFamilyCount = 0; @@ -728,19 +759,33 @@ for (uint32_t i = 0; i < physdevCount; ++i) { uint32_t deviceScore = 0; - if (deviceInfos[i].hasGraphicsQueueFamily) deviceScore += 100; - if (deviceInfos[i].supportsExternalMemory) deviceScore += 10; - if (deviceInfos[i].hasComputeQueueFamily) deviceScore += 1; + if (deviceInfos[i].hasGraphicsQueueFamily) deviceScore += 10000; + if (deviceInfos[i].supportsExternalMemory) deviceScore += 1000; + if (deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || + deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU) { + deviceScore += 100; + } + if (deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) { + deviceScore += 50; + } deviceScores[i] = deviceScore; } uint32_t maxScoringIndex = 0; uint32_t maxScore = 0; - for (uint32_t i = 0; i < physdevCount; ++i) { - if (deviceScores[i] > maxScore) { - maxScoringIndex = i; - maxScore = deviceScores[i]; + // If we don't support physical device ID properties, + // just pick the first physical device. + if (!sVkEmulation->instanceSupportsExternalMemoryCapabilities) { + fprintf(stderr, "%s: warning: instance doesn't support " + "external memory capabilities, picking first physical device\n", __func__); + maxScoringIndex = 0; + } else { + for (uint32_t i = 0; i < physdevCount; ++i) { + if (deviceScores[i] > maxScore) { + maxScoringIndex = i; + maxScore = deviceScores[i]; + } } }
diff --git a/stream-servers/vulkan/VkCommonOperations.h b/stream-servers/vulkan/VkCommonOperations.h index 14e2a17..5a8fb67 100644 --- a/stream-servers/vulkan/VkCommonOperations.h +++ b/stream-servers/vulkan/VkCommonOperations.h
@@ -83,6 +83,8 @@ bool instanceSupportsExternalMemoryCapabilities = false; PFN_vkGetPhysicalDeviceImageFormatProperties2KHR getImageFormatProperties2Func = nullptr; + PFN_vkGetPhysicalDeviceProperties2KHR + getPhysicalDeviceProperties2Func = nullptr; bool instanceSupportsMoltenVK = false; PFN_vkSetMTLTextureMVK setMTLTextureFunc = nullptr; @@ -125,6 +127,7 @@ bool hasGraphicsQueueFamily = false; bool hasComputeQueueFamily = false; bool supportsExternalMemory = false; + bool supportsIdProperties = false; bool glInteropSupported = false; std::vector<uint32_t> graphicsQueueFamilyIndices; @@ -132,6 +135,7 @@ VkPhysicalDeviceProperties physdevProps; VkPhysicalDeviceMemoryProperties memProps; + VkPhysicalDeviceIDPropertiesKHR idProps; PFN_vkGetImageMemoryRequirements2KHR getImageMemoryRequirements2Func = nullptr; PFN_vkGetBufferMemoryRequirements2KHR getBufferMemoryRequirements2Func = nullptr;
diff --git a/stream-servers/vulkan/VkDecoderGlobalState.cpp b/stream-servers/vulkan/VkDecoderGlobalState.cpp index 2521a9d..3bf7eca 100644 --- a/stream-servers/vulkan/VkDecoderGlobalState.cpp +++ b/stream-servers/vulkan/VkDecoderGlobalState.cpp
@@ -555,6 +555,8 @@ AutoLock lock(mLock); + VkPhysicalDevice advertisedPhysicalDevice = VK_NULL_HANDLE; + if (physicalDeviceCount && physicalDevices) { // Box them up for (uint32_t i = 0; i < *physicalDeviceCount; ++i) { @@ -562,7 +564,6 @@ auto& physdevInfo = mPhysdevInfo[physicalDevices[i]]; - physdevInfo.boxed = new_boxed_VkPhysicalDevice(physicalDevices[i], vk, false /* does not own dispatch */); @@ -588,7 +589,64 @@ physicalDevices[i], &queueFamilyPropCount, physdevInfo.queueFamilyProperties.data()); + VkPhysicalDevice physdevForIdQuery = physicalDevices[i]; physicalDevices[i] = (VkPhysicalDevice)physdevInfo.boxed; + + if (m_emu->instanceSupportsExternalMemoryCapabilities) { + PFN_vkGetPhysicalDeviceProperties2KHR getPhysdevProps2Func = (PFN_vkGetPhysicalDeviceProperties2KHR)( + vk->vkGetInstanceProcAddr( + instance, "vkGetPhysicalDeviceProperties2KHR")); + + if (!getPhysdevProps2Func) { + getPhysdevProps2Func = (PFN_vkGetPhysicalDeviceProperties2KHR)( + vk->vkGetInstanceProcAddr( + instance, "vkGetPhysicalDeviceProperties2")); + } + + if (!getPhysdevProps2Func) { + PFN_vkGetPhysicalDeviceProperties2KHR khrFunc = vk->vkGetPhysicalDeviceProperties2KHR; + PFN_vkGetPhysicalDeviceProperties2KHR coreFunc = vk->vkGetPhysicalDeviceProperties2; + + if (coreFunc) getPhysdevProps2Func = coreFunc; + if (!getPhysdevProps2Func && khrFunc) getPhysdevProps2Func = khrFunc; + } + + if (getPhysdevProps2Func) { + // We can get the device UUID. + VkPhysicalDeviceIDPropertiesKHR idProps = { + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR, nullptr, + }; + VkPhysicalDeviceProperties2KHR propsWithId = { + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR, &idProps, + }; + getPhysdevProps2Func(physdevForIdQuery, &propsWithId); + + // The device UUID matches the one in VkCommonOperations. + if (0 == memcmp(m_emu->deviceInfo.idProps.deviceUUID, idProps.deviceUUID, VK_UUID_SIZE)) { + advertisedPhysicalDevice = physicalDevices[i]; + } + } else { + fprintf(stderr, "%s: warning: failed to vkGetPhysicalDeviceProperties2KHR\n", __func__); + } + } else { + // If we don't support ID properties then just advertise only the first physical device. + fprintf(stderr, "%s: device id properties not supported, using first physical device\n", __func__); + advertisedPhysicalDevice = physicalDevices[0]; + } + } + } + + if (physicalDeviceCount && (*physicalDeviceCount > 0)) { + // Hide all physical devices except for the advertised one (the one whose UUID matches + // the one in VkCommonOperations.cpp) + *physicalDeviceCount = 1; + + if (physicalDevices) { + if (!advertisedPhysicalDevice) { + *physicalDevices = physicalDevices[0]; + } else { + *physicalDevices = advertisedPhysicalDevice; + } } } @@ -4802,6 +4860,7 @@ #endif } } + return res; }