Move back RenderDoc.h from aemu-common.

Bug: 246990929
Test: presubmit
Change-Id: I404ae4ef278dde1a6d24c737babf98b5286a7384
diff --git a/Android.bp b/Android.bp
index 18dd1e9..8ba628d 100644
--- a/Android.bp
+++ b/Android.bp
@@ -54,6 +54,7 @@
         "stream-servers/gl",
         "stream-servers/gl/glestranslator/include",
         "stream-servers/vulkan",
+        "utils/include",
     ],
     header_libs: [
         "aemu_common_headers",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6630124..d955c51 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,6 +127,7 @@
 
 # Backends######################################################################
 
+add_subdirectory(utils)
 add_subdirectory(stream-servers)
 
 # Protocols and associated code generators######################################
diff --git a/stream-servers/FrameBuffer.h b/stream-servers/FrameBuffer.h
index 5c062d7..d3c9473 100644
--- a/stream-servers/FrameBuffer.h
+++ b/stream-servers/FrameBuffer.h
@@ -53,11 +53,11 @@
 #include "gl/GLESVersionDetector.h"
 #include "gl/RenderContext.h"
 #include "gl/TextureDraw.h"
-#include "host-common/RenderDoc.h"
 #include "render_api.h"
 #include "render-utils/Renderer.h"
 #include "render-utils/virtio_gpu_ops.h"
 #include "snapshot/common.h"
+#include "utils/RenderDoc.h"
 #include "vulkan/vk_util.h"
 
 using android::base::CreateMetricsLogger;
diff --git a/stream-servers/vulkan/CMakeLists.txt b/stream-servers/vulkan/CMakeLists.txt
index 3af7c68..e60c2bf 100644
--- a/stream-servers/vulkan/CMakeLists.txt
+++ b/stream-servers/vulkan/CMakeLists.txt
@@ -28,6 +28,7 @@
         apigen-codec-common
         aemu-base.headers
         gfxstream-snapshot.headers
+        gfxstream_utils.headers
         gfxstream_vulkan_headers)
 
 if (WIN32)
diff --git a/stream-servers/vulkan/VkCommonOperations.h b/stream-servers/vulkan/VkCommonOperations.h
index daefa79..6fa906d 100644
--- a/stream-servers/vulkan/VkCommonOperations.h
+++ b/stream-servers/vulkan/VkCommonOperations.h
@@ -29,7 +29,7 @@
 #include "aemu/base/ManagedDescriptor.hpp"
 #include "aemu/base/Optional.h"
 #include "cereal/common/goldfish_vk_private_defs.h"
-#include "host-common/RenderDoc.h"
+#include "utils/RenderDoc.h"
 
 namespace goldfish_vk {
 
diff --git a/stream-servers/vulkan/VkDecoderGlobalState.cpp b/stream-servers/vulkan/VkDecoderGlobalState.cpp
index c771ea0..e52201d 100644
--- a/stream-servers/vulkan/VkDecoderGlobalState.cpp
+++ b/stream-servers/vulkan/VkDecoderGlobalState.cpp
@@ -49,10 +49,10 @@
 #include "compressedTextureFormats/etc.h"
 #include "host-common/GfxstreamFatalError.h"
 #include "host-common/HostmemIdMapping.h"
-#include "host-common/RenderDoc.h"
 #include "host-common/address_space_device_control_ops.h"
 #include "host-common/feature_control.h"
 #include "host-common/vm_operations.h"
+#include "utils/RenderDoc.h"
 #include "vk_util.h"
 #include "vulkan/emulated_textures/AstcTexture.h"
 #include "vulkan/vk_enum_string_helper.h"
diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt
index 3baffe2..0dbc1fe 100644
--- a/third-party/CMakeLists.txt
+++ b/third-party/CMakeLists.txt
@@ -46,8 +46,6 @@
             gfxstream_snapshot)
         foreach(PC_FILE IN LISTS AEMU_PC_FILES)
             pkg_search_module(${PC_FILE} REQUIRED IMPORTED_TARGET GLOBAL ${PC_FILE}>=0.0.0)
-#           string(REPLACE "_" "-" LIB_NAME "${PC_FILE}")
-#           add_library(${LIB_NAME} ALIAS PkgConfig::${PC_FILE})
         endforeach()
 
         add_library(aemu-base.headers INTERFACE)
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
new file mode 100644
index 0000000..4091c91
--- /dev/null
+++ b/utils/CMakeLists.txt
@@ -0,0 +1,28 @@
+add_library(gfxstream_utils.headers INTERFACE)
+
+target_include_directories(gfxstream_utils.headers INTERFACE include)
+target_link_libraries(
+    gfxstream_utils.headers
+    INTERFACE
+    renderdoc
+    gfxstream_vulkan_headers
+    aemu-host-common.headers
+    aemu-base.headers)
+
+if (ENABLE_VKCEREAL_TESTS)
+    add_executable(
+        gfxstream_utils_unittests
+        RenderDoc_unittest.cpp)
+
+    target_link_libraries(
+        gfxstream_utils_unittests
+        PRIVATE
+        gfxstream_utils.headers
+        ${GFXSTREAM_HOST_COMMON_LIB}
+        ${GFXSTREAM_BASE_LIB}
+        renderdoc
+        gtest_main
+        gmock_main)
+
+    gtest_discover_tests(gfxstream_utils_unittests)
+endif()
diff --git a/utils/RenderDoc_unittest.cpp b/utils/RenderDoc_unittest.cpp
new file mode 100644
index 0000000..348a99e
--- /dev/null
+++ b/utils/RenderDoc_unittest.cpp
@@ -0,0 +1,178 @@
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "utils/RenderDoc.h"
+
+#include <type_traits>
+
+#include "aemu/base/SharedLibrary.h"
+
+namespace emugl {
+namespace {
+
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::MockFunction;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+
+using FunctionPtr = android::base::SharedLibrary::FunctionPtr;
+using RenderDocApi = RenderDoc::RenderDocApi;
+
+class MockSharedLibrary : public android::base::SharedLibrary {
+   public:
+    MockSharedLibrary() : SharedLibrary(NULL) {}
+    MOCK_METHOD(FunctionPtr, findSymbol, (const char*), (const, override));
+};
+
+TEST(RenderDocTest, InitializeWithNullSharedLibrary) {
+    EXPECT_EQ(RenderDoc::create(nullptr), nullptr);
+}
+
+TEST(RenderDocTest, CantFindRENDERDOC_GetAPI) {
+    MockSharedLibrary sharedLibrary;
+    EXPECT_CALL(sharedLibrary, findSymbol(_)).WillRepeatedly(Return(nullptr));
+    EXPECT_EQ(RenderDoc::create(&sharedLibrary), nullptr);
+}
+
+TEST(RenderDocTest, RENDERDOC_GetAPIFails) {
+    MockSharedLibrary sharedLibrary;
+    static MockFunction<std::remove_pointer_t<pRENDERDOC_GetAPI>> mockRENDERDOC_GetAPI;
+    static pRENDERDOC_GetAPI fpMockRENDERDOC_GetAPI = [](RENDERDOC_Version version,
+                                                         void** outAPIPointers) {
+        return mockRENDERDOC_GetAPI.Call(version, outAPIPointers);
+    };
+    RenderDocApi rdocApi;
+
+    EXPECT_CALL(sharedLibrary, findSymbol(_)).WillRepeatedly(Return(nullptr));
+    EXPECT_CALL(sharedLibrary, findSymbol("RENDERDOC_GetAPI"))
+        .WillRepeatedly(Return(reinterpret_cast<FunctionPtr>(fpMockRENDERDOC_GetAPI)));
+
+    EXPECT_CALL(mockRENDERDOC_GetAPI, Call(_, _))
+        .WillRepeatedly(DoAll(SetArgPointee<1>(&rdocApi), Return(0)));
+    EXPECT_EQ(RenderDoc::create(&sharedLibrary), nullptr);
+
+    EXPECT_CALL(mockRENDERDOC_GetAPI, Call(_, _))
+        .WillRepeatedly(DoAll(SetArgPointee<1>(nullptr), Return(1)));
+    EXPECT_EQ(RenderDoc::create(&sharedLibrary), nullptr);
+}
+
+TEST(RenderDocTest, CreateSuccessfully) {
+    MockSharedLibrary sharedLibrary;
+    static MockFunction<std::remove_pointer_t<pRENDERDOC_GetAPI>> mockRENDERDOC_GetAPI;
+    static pRENDERDOC_GetAPI fpMockRENDERDOC_GetAPI = [](RENDERDOC_Version version,
+                                                         void** outAPIPointers) {
+        return mockRENDERDOC_GetAPI.Call(version, outAPIPointers);
+    };
+    RenderDocApi rdocApiMock;
+    static MockFunction<std::remove_pointer_t<pRENDERDOC_GetCaptureOptionU32>>
+        getCaptureOptionU32Mock;
+    rdocApiMock.GetCaptureOptionU32 = [](RENDERDOC_CaptureOption option) {
+        return getCaptureOptionU32Mock.Call(option);
+    };
+
+    EXPECT_CALL(sharedLibrary, findSymbol(_)).WillRepeatedly(Return(nullptr));
+    EXPECT_CALL(sharedLibrary, findSymbol("RENDERDOC_GetAPI"))
+        .WillRepeatedly(Return(reinterpret_cast<FunctionPtr>(fpMockRENDERDOC_GetAPI)));
+    EXPECT_CALL(mockRENDERDOC_GetAPI, Call(_, _))
+        .WillRepeatedly(DoAll(SetArgPointee<1>(&rdocApiMock), Return(1)));
+    std::unique_ptr<RenderDoc> renderDoc = RenderDoc::create(&sharedLibrary);
+    EXPECT_NE(renderDoc, nullptr);
+
+    EXPECT_CALL(getCaptureOptionU32Mock, Call(eRENDERDOC_Option_DebugOutputMute))
+        .WillRepeatedly(Return(1));
+    EXPECT_EQ(renderDoc->call(RenderDoc::kGetCaptureOptionU32, eRENDERDOC_Option_DebugOutputMute),
+              1);
+}
+
+class RenderDocMock {
+   public:
+    // For StartFrameCapture
+    MOCK_METHOD(void, call,
+                (void (*RenderDocApi::*function)(RENDERDOC_DevicePointer, RENDERDOC_WindowHandle),
+                 RENDERDOC_DevicePointer, RENDERDOC_WindowHandle),
+                (const));
+    // For EndFrameCapture
+    MOCK_METHOD(uint32_t, call,
+                (uint32_t(*RenderDocApi::*function)(RENDERDOC_DevicePointer,
+                                                    RENDERDOC_WindowHandle),
+                 RENDERDOC_DevicePointer, RENDERDOC_WindowHandle),
+                (const));
+    // For IsFrameCapturing
+    MOCK_METHOD(uint32_t, call, (uint32_t(*RenderDocApi::*function)()), (const));
+};
+
+using RenderDocWithMultipleVkInstances = RenderDocWithMultipleVkInstancesBase<RenderDocMock>;
+
+TEST(RenderDocWithMultipleVkInstancesTest,
+     ShouldNotStartFrameCaptureOnFrameDelimiterWhenNotCapturing) {
+    RenderDocMock renderDocMock;
+    intptr_t vkInstanceInternal = 0x1234;
+    VkInstance vkInstance = reinterpret_cast<VkInstance>(&vkInstanceInternal);
+    RenderDocWithMultipleVkInstances renderDocWithMultipleVkInstances(renderDocMock);
+
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kIsFrameCapturing)).WillRepeatedly(Return(0));
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kStartFrameCapture, _, _)).Times(0);
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kEndFrameCapture, _, _)).Times(0);
+
+    renderDocWithMultipleVkInstances.onFrameDelimiter(vkInstance);
+}
+
+TEST(RenderDocWithMultipleVkInstancesTest, ShouldStartAndEndFrameCaptureOnFrameDelimiter) {
+    RenderDocMock renderDocMock;
+    intptr_t vkInstanceInternal = 0x4321;
+    VkInstance vkInstance = reinterpret_cast<VkInstance>(&vkInstanceInternal);
+    RenderDocWithMultipleVkInstances renderDocWithMultipleVkInstances(renderDocMock);
+
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kIsFrameCapturing)).WillRepeatedly(Return(1));
+    {
+        InSequence s;
+
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kStartFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1);
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kEndFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1)
+            .WillRepeatedly(Return(1));
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kStartFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1);
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kEndFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1)
+            .WillRepeatedly(Return(1));
+    }
+
+    renderDocWithMultipleVkInstances.onFrameDelimiter(vkInstance);
+    renderDocWithMultipleVkInstances.onFrameDelimiter(vkInstance);
+}
+
+TEST(RenderDocWithMultipleVkInstancesTest, ShouldEndFrameCaptureOnVkInstanceRemoved) {
+    RenderDocMock renderDocMock;
+    intptr_t vkInstanceInternal = 0x4321;
+    VkInstance vkInstance = reinterpret_cast<VkInstance>(&vkInstanceInternal);
+    RenderDocWithMultipleVkInstances renderDocWithMultipleVkInstances(renderDocMock);
+
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kIsFrameCapturing)).WillRepeatedly(Return(1));
+    {
+        InSequence s;
+
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kStartFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1);
+        EXPECT_CALL(renderDocMock, call(RenderDoc::kEndFrameCapture,
+                                        RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+            .Times(1)
+            .WillRepeatedly(Return(1));
+    }
+
+    renderDocWithMultipleVkInstances.onFrameDelimiter(vkInstance);
+    renderDocWithMultipleVkInstances.removeVkInstance(vkInstance);
+    EXPECT_CALL(renderDocMock, call(RenderDoc::kEndFrameCapture,
+                                    RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vkInstance), NULL))
+        .Times(0);
+}
+}  // namespace
+}  // namespace emugl
diff --git a/utils/include/utils/RenderDoc.h b/utils/include/utils/RenderDoc.h
new file mode 100644
index 0000000..f3eacd5
--- /dev/null
+++ b/utils/include/utils/RenderDoc.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2011-2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RENDER_DOC_H
+#define RENDER_DOC_H
+
+#include <renderdoc_app.h>
+
+#include <algorithm>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+
+#include "aemu/base/SharedLibrary.h"
+#include "host-common/logging.h"
+#include "vulkan/vulkan.h"
+
+using android::base::SharedLibrary;
+
+namespace emugl {
+class RenderDoc {
+   public:
+    using RenderDocApi = RENDERDOC_API_1_4_2;
+    static std::unique_ptr<RenderDoc> create(const SharedLibrary* renderDocLib) {
+        if (!renderDocLib) {
+            ERR("The renderdoc shared library is null.");
+            return nullptr;
+        }
+        pRENDERDOC_GetAPI RENDERDOC_GetAPI =
+            reinterpret_cast<pRENDERDOC_GetAPI>(renderDocLib->findSymbol("RENDERDOC_GetAPI"));
+        if (!RENDERDOC_GetAPI) {
+            ERR("Failed to find the RENDERDOC_GetAPI symbol.");
+            return nullptr;
+        }
+        RenderDocApi* rdocApi = nullptr;
+        int ret =
+            RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_4_2, reinterpret_cast<void**>(&rdocApi));
+        if (ret != 1 || rdocApi == nullptr) {
+            ERR("Failed to load renderdoc API. %d is returned from RENDERDOC_GetAPI.");
+            return nullptr;
+        }
+        return std::unique_ptr<RenderDoc>(new RenderDoc(rdocApi));
+    }
+
+    static constexpr auto kSetActiveWindow = &RenderDocApi::SetActiveWindow;
+    static constexpr auto kGetCaptureOptionU32 = &RenderDocApi::GetCaptureOptionU32;
+    static constexpr auto kIsFrameCapturing = &RenderDocApi::IsFrameCapturing;
+    static constexpr auto kStartFrameCapture = &RenderDocApi::StartFrameCapture;
+    static constexpr auto kEndFrameCapture = &RenderDocApi::EndFrameCapture;
+    template <class F, class... Args, typename = std::enable_if_t<std::is_invocable_v<F, Args...>>>
+    // Call a RenderDoc in-application API given the function pointer and parameters, and guard the
+    // API call with a mutex.
+    auto call(F(RenderDocApi::*function), Args... args) const {
+        std::lock_guard<std::mutex> guard(mMutex);
+        return (mRdocApi->*function)(args...);
+    }
+
+   private:
+    RenderDoc(RenderDocApi* rdocApi) : mRdocApi(rdocApi) {}
+
+    mutable std::mutex mMutex;
+    RenderDocApi* mRdocApi = nullptr;
+};
+
+template <class RenderDocT>
+class RenderDocWithMultipleVkInstancesBase {
+   public:
+    RenderDocWithMultipleVkInstancesBase(RenderDocT& renderDoc) : mRenderDoc(renderDoc) {}
+
+    void onFrameDelimiter(VkInstance vkInstance) {
+        std::lock_guard<std::mutex> guard(mMutex);
+        mCaptureContexts.erase(vkInstance);
+        if (mRenderDoc.call(RenderDoc::kIsFrameCapturing)) {
+            mCaptureContexts.emplace(vkInstance,
+                                     std::make_unique<CaptureContext>(mRenderDoc, vkInstance));
+        }
+    }
+
+    void removeVkInstance(VkInstance vkInstance) {
+        std::lock_guard<std::mutex> guard(mMutex);
+        mCaptureContexts.erase(vkInstance);
+    }
+
+   private:
+    class CaptureContext {
+       public:
+        CaptureContext(RenderDocT& renderDoc, VkInstance vkInstance)
+            : mRenderDoc(renderDoc), mVkInstance(vkInstance) {
+            mRenderDoc.call(RenderDoc::kStartFrameCapture,
+                            RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(mVkInstance), nullptr);
+        }
+        ~CaptureContext() {
+            mRenderDoc.call(RenderDoc::kEndFrameCapture,
+                            RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(mVkInstance), nullptr);
+        }
+
+       private:
+        const RenderDocT& mRenderDoc;
+        const VkInstance mVkInstance;
+    };
+    std::mutex mMutex;
+    std::unordered_map<VkInstance, std::unique_ptr<CaptureContext>> mCaptureContexts;
+    RenderDocT& mRenderDoc;
+};
+
+using RenderDocWithMultipleVkInstances = RenderDocWithMultipleVkInstancesBase<RenderDoc>;
+}  // namespace emugl
+
+#endif