vulkan/gles interop: ensure consistent selection of physdevs / interop
This should solve two potential issues:
1. Use VkPhysicalDeviceIDProperties to advertise only the physical
device we selected to the guest. If we ended up selecting a different
physical device in VkCommonOperations versus on_vkCreateDevice from the
guest, problems can happen.
2. Ensure device UUID between gles/vk match. If they don't, we can't use
interop, even if the gl side reports support for the interop extensions
(think NV gl alongside Intel vk). The gles side should have a way to
query the UUID of the device (tested with native GL and GLES on Linux
and native GL on Windows); otherwise, we can't use zero-copy interop.
Bug: 190561902
Change-Id: I6c6e217a3c09566ab4c0d47a91432c58343efc54
diff --git a/stream-servers/FrameBuffer.cpp b/stream-servers/FrameBuffer.cpp
index b9966e0..daef6c0 100644
--- a/stream-servers/FrameBuffer.cpp
+++ b/stream-servers/FrameBuffer.cpp
@@ -354,20 +354,29 @@
// used by underlying EGL driver might become invalid,
// preventing new contexts from being created that share
// against those contexts.
+ goldfish_vk::VkEmulation* vkEmu = nullptr;
if (feature_is_enabled(kFeature_Vulkan)) {
auto dispatch = emugl::vkDispatch(false /* not for testing */);
- auto emu = goldfish_vk::createOrGetGlobalVkEmulation(dispatch);
+ vkEmu = goldfish_vk::createOrGetGlobalVkEmulation(dispatch);
bool useDeferredCommands =
android::base::getEnvironmentVariable("ANDROID_EMU_VK_DISABLE_DEFERRED_COMMANDS").empty();
bool useCreateResourcesWithRequirements =
android::base::getEnvironmentVariable("ANDROID_EMU_VK_DISABLE_USE_CREATE_RESOURCES_WITH_REQUIREMENTS").empty();
- goldfish_vk::setUseDeferredCommands(emu, useDeferredCommands);
- goldfish_vk::setUseCreateResourcesWithRequirements(emu, useCreateResourcesWithRequirements);
+ goldfish_vk::setUseDeferredCommands(vkEmu, useDeferredCommands);
+ goldfish_vk::setUseCreateResourcesWithRequirements(vkEmu, useCreateResourcesWithRequirements);
if (feature_is_enabled(kFeature_VulkanNativeSwapchain)) {
fb->m_displayVk = std::make_unique<DisplayVk>(
- *dispatch, emu->physdev, emu->queueFamilyIndex,
- emu->queueFamilyIndex, emu->device, emu->queue, emu->queue);
- fb->m_vkInstance = emu->instance;
+ *dispatch, vkEmu->physdev, vkEmu->queueFamilyIndex,
+ vkEmu->queueFamilyIndex, vkEmu->device, vkEmu->queue, vkEmu->queue);
+ fb->m_vkInstance = vkEmu->instance;
+ }
+ if (vkEmu->deviceInfo.supportsIdProperties) {
+ GL_LOG("Supports id properties, got a vulkan device UUID");
+ fprintf(stderr, "%s: Supports id properties, got a vulkan device UUID\n", __func__);
+ memcpy(fb->m_vulkanUUID, vkEmu->deviceInfo.idProps.deviceUUID, VK_UUID_SIZE);
+ } else {
+ GL_LOG("Doesn't support id properties, no vulkan device UUID");
+ fprintf(stderr, "%s: Doesn't support id properties, no vulkan device UUID\n", __func__);
}
}
@@ -624,6 +633,47 @@
fb->m_glRenderer = std::string((const char*)s_gles2.glGetString(GL_RENDERER));
fb->m_glVersion = std::string((const char*)s_gles2.glGetString(GL_VERSION));
+ // Attempt to get the device UUID of the gles and match with Vulkan. If
+ // they match, interop is possible. If they don't, then don't trust the
+ // result of interop query to egl and fall back to CPU copy, as we might
+ // have initialized Vulkan devices and GLES contexts from different
+ // physical devices.
+
+ bool vkglesUuidsGood = true;
+
+ // First, if the VkEmulation instance doesn't support ext memory capabilities,
+ // it won't support uuids.
+ if (!vkEmu || !vkEmu->deviceInfo.supportsIdProperties) {
+ vkglesUuidsGood = false;
+ }
+
+ s_gles2.glGetError();
+
+ GLint numDeviceUuids = 0;
+ s_gles2.glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &numDeviceUuids);
+
+ // If underlying gles doesn't support UUID query, we definitely don't
+ // support interop and should not proceed further.
+
+ if (!numDeviceUuids || 1 != numDeviceUuids) {
+ // If numDeviceUuids != 1 it's unclear what gles we're using (SLI? Xinerama?)
+ // and we shouldn't try to interop.
+ vkglesUuidsGood = false;
+ }
+
+ if (vkglesUuidsGood && 1 == numDeviceUuids) {
+ s_gles2.glGetUnsignedBytei_vEXT(GL_DEVICE_UUID_EXT, 0, fb->m_glesUUID);
+ GL_LOG("Underlying gles supports UUID");
+ if (0 == memcmp(fb->m_vulkanUUID, fb->m_glesUUID, VK_UUID_SIZE)) {
+ GL_LOG("vk/gles UUIDs match");
+ fprintf(stderr, "%s: vk/gles UUIDs match\n", __func__);
+ } else {
+ GL_LOG("vk/gles UUIDs do not match");
+ fprintf(stderr, "%s: vk/gles UUIDs do not match\n", __func__);
+ vkglesUuidsGood = false;
+ }
+ }
+
DBG("GL Vendor %s\n", fb->m_glVendor.c_str());
DBG("GL Renderer %s\n", fb->m_glRenderer.c_str());
DBG("GL Extensions %s\n", fb->m_glVersion.c_str());
@@ -641,6 +691,9 @@
if (s_egl.eglQueryVulkanInteropSupportANDROID) {
fb->m_vulkanInteropSupported =
s_egl.eglQueryVulkanInteropSupportANDROID();
+ if (!vkglesUuidsGood) {
+ fb->m_vulkanInteropSupported = false;
+ }
}
fprintf(stderr, "%s: interop? %d\n", __func__, fb->m_vulkanInteropSupported);
@@ -768,6 +821,8 @@
setDisplayPose(displayId, 0, 0, getWidth(), getHeight(), 0);
m_perfThread->start();
+ memset(m_vulkanUUID, 0x0, 16);
+ memset(m_glesUUID, 0x0, 16);
}
FrameBuffer::~FrameBuffer() {
diff --git a/stream-servers/FrameBuffer.h b/stream-servers/FrameBuffer.h
index 42ff034..79353ba 100644
--- a/stream-servers/FrameBuffer.h
+++ b/stream-servers/FrameBuffer.h
@@ -780,5 +780,10 @@
std::unique_ptr<DisplayVk> m_displayVk;
VkInstance m_vkInstance = VK_NULL_HANDLE;
VkSurfaceKHR m_vkSurface = VK_NULL_HANDLE;
+
+ // UUIDs of physical devices for Vulkan and GLES, respectively. In most
+ // cases, this determines whether we can support zero-copy interop.
+ uint8_t m_vulkanUUID[VK_UUID_SIZE];
+ uint8_t m_glesUUID[VK_UUID_SIZE];
};
#endif
diff --git a/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp b/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp
index b31fa34..a8cfd56 100644
--- a/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp
+++ b/stream-servers/glestranslator/GLES_V2/GLESv2Imp.cpp
@@ -4476,11 +4476,13 @@
// Common between GL_EXT_memory_object and GL_EXT_semaphore
GL_APICALL void GL_APIENTRY glGetUnsignedBytevEXT(GLenum pname, GLubyte* data) {
GET_CTX_V2();
+ if (!ctx->dispatcher().glGetUnsignedBytevEXT) return;
ctx->dispatcher().glGetUnsignedBytevEXT(pname, data);
}
GL_APICALL void GL_APIENTRY glGetUnsignedBytei_vEXT(GLenum target, GLuint index, GLubyte* data) {
GET_CTX_V2();
+ if (!ctx->dispatcher().glGetUnsignedBytei_vEXT) return;
ctx->dispatcher().glGetUnsignedBytei_vEXT(target, index, data);
}
diff --git a/stream-servers/vulkan/VkCommonOperations.cpp b/stream-servers/vulkan/VkCommonOperations.cpp
index b0b37bf..18bda70 100644
--- a/stream-servers/vulkan/VkCommonOperations.cpp
+++ b/stream-servers/vulkan/VkCommonOperations.cpp
@@ -602,6 +602,18 @@
ivk->vkGetInstanceProcAddr(
sVkEmulation->instance,
"vkGetPhysicalDeviceImageFormatProperties2KHR"));
+ sVkEmulation->getPhysicalDeviceProperties2Func = reinterpret_cast<
+ PFN_vkGetPhysicalDeviceProperties2KHR>(
+ ivk->vkGetInstanceProcAddr(
+ sVkEmulation->instance,
+ "vkGetPhysicalDeviceProperties2KHR"));
+ if (!sVkEmulation->getPhysicalDeviceProperties2Func) {
+ sVkEmulation->getPhysicalDeviceProperties2Func = reinterpret_cast<
+ PFN_vkGetPhysicalDeviceProperties2KHR>(
+ ivk->vkGetInstanceProcAddr(
+ sVkEmulation->instance,
+ "vkGetPhysicalDeviceProperties2"));
+ }
}
if (sVkEmulation->instanceSupportsMoltenVK) {
@@ -664,6 +676,25 @@
if (sVkEmulation->instanceSupportsExternalMemoryCapabilities) {
deviceInfos[i].supportsExternalMemory = extensionsSupported(
deviceExts, externalMemoryDeviceExtNames);
+ deviceInfos[i].supportsIdProperties =
+ sVkEmulation->getPhysicalDeviceProperties2Func != nullptr;
+ if (!sVkEmulation->getPhysicalDeviceProperties2Func) {
+ fprintf(stderr, "%s: warning: device should support ID properties "
+ "but vkGetPhysicalDeviceProperties2 could not be found\n", __func__);
+ }
+ }
+
+ if (deviceInfos[i].supportsIdProperties) {
+ VkPhysicalDeviceIDPropertiesKHR idProps = {
+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR, nullptr,
+ };
+ VkPhysicalDeviceProperties2KHR propsWithId = {
+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR, &idProps,
+ };
+ sVkEmulation->getPhysicalDeviceProperties2Func(
+ physdevs[i],
+ &propsWithId);
+ deviceInfos[i].idProps = idProps;
}
uint32_t queueFamilyCount = 0;
@@ -728,19 +759,33 @@
for (uint32_t i = 0; i < physdevCount; ++i) {
uint32_t deviceScore = 0;
- if (deviceInfos[i].hasGraphicsQueueFamily) deviceScore += 100;
- if (deviceInfos[i].supportsExternalMemory) deviceScore += 10;
- if (deviceInfos[i].hasComputeQueueFamily) deviceScore += 1;
+ if (deviceInfos[i].hasGraphicsQueueFamily) deviceScore += 10000;
+ if (deviceInfos[i].supportsExternalMemory) deviceScore += 1000;
+ if (deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ||
+ deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU) {
+ deviceScore += 100;
+ }
+ if (deviceInfos[i].physdevProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) {
+ deviceScore += 50;
+ }
deviceScores[i] = deviceScore;
}
uint32_t maxScoringIndex = 0;
uint32_t maxScore = 0;
- for (uint32_t i = 0; i < physdevCount; ++i) {
- if (deviceScores[i] > maxScore) {
- maxScoringIndex = i;
- maxScore = deviceScores[i];
+ // If we don't support physical device ID properties,
+ // just pick the first physical device.
+ if (!sVkEmulation->instanceSupportsExternalMemoryCapabilities) {
+ fprintf(stderr, "%s: warning: instance doesn't support "
+ "external memory capabilities, picking first physical device\n", __func__);
+ maxScoringIndex = 0;
+ } else {
+ for (uint32_t i = 0; i < physdevCount; ++i) {
+ if (deviceScores[i] > maxScore) {
+ maxScoringIndex = i;
+ maxScore = deviceScores[i];
+ }
}
}
diff --git a/stream-servers/vulkan/VkCommonOperations.h b/stream-servers/vulkan/VkCommonOperations.h
index 14e2a17..5a8fb67 100644
--- a/stream-servers/vulkan/VkCommonOperations.h
+++ b/stream-servers/vulkan/VkCommonOperations.h
@@ -83,6 +83,8 @@
bool instanceSupportsExternalMemoryCapabilities = false;
PFN_vkGetPhysicalDeviceImageFormatProperties2KHR
getImageFormatProperties2Func = nullptr;
+ PFN_vkGetPhysicalDeviceProperties2KHR
+ getPhysicalDeviceProperties2Func = nullptr;
bool instanceSupportsMoltenVK = false;
PFN_vkSetMTLTextureMVK setMTLTextureFunc = nullptr;
@@ -125,6 +127,7 @@
bool hasGraphicsQueueFamily = false;
bool hasComputeQueueFamily = false;
bool supportsExternalMemory = false;
+ bool supportsIdProperties = false;
bool glInteropSupported = false;
std::vector<uint32_t> graphicsQueueFamilyIndices;
@@ -132,6 +135,7 @@
VkPhysicalDeviceProperties physdevProps;
VkPhysicalDeviceMemoryProperties memProps;
+ VkPhysicalDeviceIDPropertiesKHR idProps;
PFN_vkGetImageMemoryRequirements2KHR getImageMemoryRequirements2Func = nullptr;
PFN_vkGetBufferMemoryRequirements2KHR getBufferMemoryRequirements2Func = nullptr;
diff --git a/stream-servers/vulkan/VkDecoderGlobalState.cpp b/stream-servers/vulkan/VkDecoderGlobalState.cpp
index 2521a9d..3bf7eca 100644
--- a/stream-servers/vulkan/VkDecoderGlobalState.cpp
+++ b/stream-servers/vulkan/VkDecoderGlobalState.cpp
@@ -555,6 +555,8 @@
AutoLock lock(mLock);
+ VkPhysicalDevice advertisedPhysicalDevice = VK_NULL_HANDLE;
+
if (physicalDeviceCount && physicalDevices) {
// Box them up
for (uint32_t i = 0; i < *physicalDeviceCount; ++i) {
@@ -562,7 +564,6 @@
auto& physdevInfo = mPhysdevInfo[physicalDevices[i]];
-
physdevInfo.boxed =
new_boxed_VkPhysicalDevice(physicalDevices[i], vk, false /* does not own dispatch */);
@@ -588,7 +589,64 @@
physicalDevices[i], &queueFamilyPropCount,
physdevInfo.queueFamilyProperties.data());
+ VkPhysicalDevice physdevForIdQuery = physicalDevices[i];
physicalDevices[i] = (VkPhysicalDevice)physdevInfo.boxed;
+
+ if (m_emu->instanceSupportsExternalMemoryCapabilities) {
+ PFN_vkGetPhysicalDeviceProperties2KHR getPhysdevProps2Func = (PFN_vkGetPhysicalDeviceProperties2KHR)(
+ vk->vkGetInstanceProcAddr(
+ instance, "vkGetPhysicalDeviceProperties2KHR"));
+
+ if (!getPhysdevProps2Func) {
+ getPhysdevProps2Func = (PFN_vkGetPhysicalDeviceProperties2KHR)(
+ vk->vkGetInstanceProcAddr(
+ instance, "vkGetPhysicalDeviceProperties2"));
+ }
+
+ if (!getPhysdevProps2Func) {
+ PFN_vkGetPhysicalDeviceProperties2KHR khrFunc = vk->vkGetPhysicalDeviceProperties2KHR;
+ PFN_vkGetPhysicalDeviceProperties2KHR coreFunc = vk->vkGetPhysicalDeviceProperties2;
+
+ if (coreFunc) getPhysdevProps2Func = coreFunc;
+ if (!getPhysdevProps2Func && khrFunc) getPhysdevProps2Func = khrFunc;
+ }
+
+ if (getPhysdevProps2Func) {
+ // We can get the device UUID.
+ VkPhysicalDeviceIDPropertiesKHR idProps = {
+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR, nullptr,
+ };
+ VkPhysicalDeviceProperties2KHR propsWithId = {
+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR, &idProps,
+ };
+ getPhysdevProps2Func(physdevForIdQuery, &propsWithId);
+
+ // The device UUID matches the one in VkCommonOperations.
+ if (0 == memcmp(m_emu->deviceInfo.idProps.deviceUUID, idProps.deviceUUID, VK_UUID_SIZE)) {
+ advertisedPhysicalDevice = physicalDevices[i];
+ }
+ } else {
+ fprintf(stderr, "%s: warning: failed to vkGetPhysicalDeviceProperties2KHR\n", __func__);
+ }
+ } else {
+ // If we don't support ID properties then just advertise only the first physical device.
+ fprintf(stderr, "%s: device id properties not supported, using first physical device\n", __func__);
+ advertisedPhysicalDevice = physicalDevices[0];
+ }
+ }
+ }
+
+ if (physicalDeviceCount && (*physicalDeviceCount > 0)) {
+ // Hide all physical devices except for the advertised one (the one whose UUID matches
+ // the one in VkCommonOperations.cpp)
+ *physicalDeviceCount = 1;
+
+ if (physicalDevices) {
+ if (!advertisedPhysicalDevice) {
+ *physicalDevices = physicalDevices[0];
+ } else {
+ *physicalDevices = advertisedPhysicalDevice;
+ }
}
}
@@ -4802,6 +4860,7 @@
#endif
}
}
+
return res;
}
