gfxstream: stream_renderer_snapshot(..) + stream_renderer_restore(..)

This is designed to match the API added in crrev.com/c/4853266.
The issue with the prior one is it assumes an aemu::snapshot
C++ object is passed through, which would be difficult to get
working with crosvm or upstream QEMU.

Passing through a C-string to a directory is simpler and more
portable.

In addition, since the API is only used with the testing layer,
GFXSTREAM_ENABLE_HOST_SNAPSHOT is added.  This is because the
code assumes a newer AEMU, and the versioning on that is still
TBD.  In production, STREAM_RENDERER_FLAGS_VULKAN_SNAPSHOTS will
always be true likely, so the runtime flag doesn't add much value.

BUG=324440526
TEST=End2EndTests pass

Change-Id: I71686603d8293781733fc5eb5fd936e46871baf7
diff --git a/Android.bp b/Android.bp
index a4ce1a4..cf44e07 100644
--- a/Android.bp
+++ b/Android.bp
@@ -124,6 +124,7 @@
         "-D_FILE_OFFSET_BITS=64",
         "-DVK_GFXSTREAM_STRUCTURE_TYPE_EXT",
         "-DGFXSTREAM_ENABLE_HOST_GLES=1",
+        "-DGFXSTREAM_ENABLE_HOST_VK_SNAPSHOT=1",
         "-Wno-unreachable-code-loop-increment",
         "-Wno-unused-parameter",
         "-Wno-unused-function",
diff --git a/common/end2end/Android.bp b/common/end2end/Android.bp
index 50ce851..6e293c5 100644
--- a/common/end2end/Android.bp
+++ b/common/end2end/Android.bp
@@ -27,24 +27,15 @@
     ],
     shared_libs: [
         "libandroidemu",
-        "libOpenglCodecCommon",
         "libOpenglSystemCommonWithHost",
         "libbase",
-        "libcutils",
-        "libdrm",
         "liblog",
-        "libgfxstream_backend",
         "libplatform_rutabaga_server",
     ],
     static_libs: [
-        "gfxstream_base",
-        "gfxstream_host_common",
-        "gfxstream_snapshot",
-        "gfxstream_vulkan_cereal_host",
         "libc++fs",
         "libgfxstream_guest_android_with_host",
         "libgmock",
-        "liblz4",
         "libplatform_rutabaga",
     ],
     cflags: [
diff --git a/common/end2end/GfxstreamEnd2EndTests.cpp b/common/end2end/GfxstreamEnd2EndTests.cpp
index 4c09ec9..5d698b9 100644
--- a/common/end2end/GfxstreamEnd2EndTests.cpp
+++ b/common/end2end/GfxstreamEnd2EndTests.cpp
@@ -23,15 +23,9 @@
 #include <filesystem>
 
 #include "ProcessPipe.h"
-#include "aemu/base/files/StdioStream.h"
-#include "aemu/base/system/System.h"
-#include "drm_fourcc.h"
+#include "RutabagaLayer.h"
+#include "aemu/base/Path.h"
 #include "gfxstream/RutabagaLayerTestUtils.h"
-#include "gfxstream/virtio-gpu-gfxstream-renderer-goldfish.h"
-#include "host-common/logging.h"
-#include "snapshot/TextureLoader.h"
-#include "snapshot/TextureSaver.h"
-#include "snapshot/common.h"
 
 VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE
 
@@ -90,7 +84,7 @@
 }
 
 std::unique_ptr<GfxstreamEnd2EndTest::GuestGlDispatchTable> GfxstreamEnd2EndTest::SetupGuestGl() {
-    const std::filesystem::path testDirectory = android::base::getProgramDirectory();
+    const std::filesystem::path testDirectory = gfxstream::guest::getProgramDirectory();
     const std::string eglLibPath = (testDirectory / "libEGL_emulation_with_host.so").string();
     const std::string gles2LibPath = (testDirectory / "libGLESv2_emulation_with_host.so").string();
 
@@ -135,7 +129,7 @@
 }
 
 std::unique_ptr<vkhpp::DynamicLoader> GfxstreamEnd2EndTest::SetupGuestVk() {
-    const std::filesystem::path testDirectory = android::base::getProgramDirectory();
+    const std::filesystem::path testDirectory = gfxstream::guest::getProgramDirectory();
     const std::string vkLibPath = (testDirectory / "libgfxstream_guest_vulkan_with_host.so").string();
 
     auto dl = std::make_unique<vkhpp::DynamicLoader>(vkLibPath);
@@ -163,10 +157,13 @@
 
     ASSERT_THAT(setenv("GFXSTREAM_EMULATED_VIRTIO_GPU_WITH_GL",
                        params.with_gl ? "Y" : "N", /*overwrite=*/1), Eq(0));
-    ASSERT_THAT(setenv("GFXSTREAM_EMULATED_VIRTIO_GPU_WITH_VK",
-                       params.with_vk ? "Y" : "N", /*overwrite=*/1), Eq(0));
+    ASSERT_THAT(setenv("GFXSTREAM_EMULATED_VIRTIO_GPU_WITH_VK", params.with_vk ? "Y" : "N",
+                       /*overwrite=*/1),
+                Eq(0));
     ASSERT_THAT(setenv("GFXSTREAM_EMULATED_VIRTIO_GPU_WITH_VK_SNAPSHOTS",
-                       params.with_vk_snapshot ? "Y" : "N", /*overwrite=*/1), Eq(0));
+                       params.with_vk_snapshot ? "Y" : "N",
+                       /*overwrite=*/1),
+                Eq(0));
 
     if (params.with_gl) {
         mGl = SetupGuestGl();
@@ -504,22 +501,12 @@
 }
 
 void GfxstreamEnd2EndTest::SnapshotSaveAndLoad() {
-    std::string snapshotFileName = testing::TempDir() + "snapshot.bin";
-    std::string textureFileName = testing::TempDir() + "texture.bin";
-    std::unique_ptr<android::base::StdioStream> stream(new android::base::StdioStream(
-        fopen(snapshotFileName.c_str(), "wb"), android::base::StdioStream::kOwner));
-    stream_renderer_snapshot_presave_pause();
-    android::snapshot::SnapshotSaveStream saveStream{
-        .stream = stream.get(),
-    };
-    stream_renderer_snapshot_save(&saveStream);
-    stream.reset(new android::base::StdioStream(fopen(snapshotFileName.c_str(), "rb"),
-                                                android::base::StdioStream::kOwner));
-    android::snapshot::SnapshotLoadStream loadStream{
-        .stream = stream.get(),
-    };
-    stream_renderer_snapshot_load(&loadStream);
-    stream_renderer_snapshot_postload_resume_for_testing();
+    auto directory = testing::TempDir();
+
+    std::shared_ptr<gfxstream::EmulatedVirtioGpu> emulation = gfxstream::EmulatedVirtioGpu::Get();
+
+    emulation->SnapshotSave(directory);
+    emulation->SnapshotRestore(directory);
 }
 
 }  // namespace tests
diff --git a/guest/platform/Android.bp b/guest/platform/Android.bp
index e356124..1fe5664 100644
--- a/guest/platform/Android.bp
+++ b/guest/platform/Android.bp
@@ -88,6 +88,7 @@
     export_include_dirs: [
         // TODO: restrict to just RutabagaLayer.h
         "rutabaga/include",
+        "rutabaga",
     ],
     target: {
         host: {
diff --git a/guest/platform/rutabaga/RutabagaLayer.cpp b/guest/platform/rutabaga/RutabagaLayer.cpp
index fb225f9..e3a1976 100644
--- a/guest/platform/rutabaga/RutabagaLayer.cpp
+++ b/guest/platform/rutabaga/RutabagaLayer.cpp
@@ -102,6 +102,8 @@
                                             uint32_t virglFormat);
 
     void DestroyResource(uint32_t contextId, uint32_t resourceId);
+    void SnapshotSave(std::string directory);
+    void SnapshotRestore(std::string directory);
 
     uint32_t CreateEmulatedFence();
 
@@ -160,12 +162,19 @@
     struct VirtioGpuTaskUnrefResource {
         uint32_t resourceId;
     };
+    struct VirtioGpuTaskSnapshotSave {
+        std::string directory;
+    };
+    struct VirtioGpuTaskSnapshotRestore {
+        std::string directory;
+    };
     using VirtioGpuTask =
         std::variant<VirtioGpuTaskContextAttachResource, VirtioGpuTaskContextDetachResource,
                      VirtioGpuTaskCreateBlob, VirtioGpuTaskCreateContext,
                      VirtioGpuTaskCreateResource, VirtioGpuTaskDestroyContext, VirtioGpuTaskMap,
                      VirtioGpuTaskExecBuffer, VirtioGpuTaskTransferFromHost,
-                     VirtioGpuTaskTransferToHost, VirtioGpuTaskUnrefResource>;
+                     VirtioGpuTaskTransferToHost, VirtioGpuTaskUnrefResource,
+                     VirtioGpuTaskSnapshotSave, VirtioGpuTaskSnapshotRestore>;
     struct VirtioGpuTaskWithWaitable {
         uint32_t contextId;
         VirtioGpuTask task;
@@ -187,6 +196,8 @@
     void DoTask(VirtioGpuTaskTransferToHost task);
     void DoTask(VirtioGpuTaskWithWaitable task);
     void DoTask(VirtioGpuTaskUnrefResource task);
+    void DoTask(VirtioGpuTaskSnapshotSave task);
+    void DoTask(VirtioGpuTaskSnapshotRestore task);
 
     void RunVirtioGpuTaskProcessingLoop();
 
@@ -279,6 +290,13 @@
 
 bool EmulatedVirtioGpu::EmulatedVirtioGpuImpl::Init(bool withGl, bool withVk, bool withVkSnapshots,
                                                     EmulatedVirtioGpu* parent) {
+    int ret = setenv("ANDROID_GFXSTREAM_CAPTURE_VK_SNAPSHOT", withVkSnapshots ? "1" : "0",
+                     1 /* replace */);
+    if (ret) {
+        ALOGE("Failed to set environment variable");
+        return false;
+    }
+
     std::vector<stream_renderer_param> renderer_params{
         stream_renderer_param{
             .key = STREAM_RENDERER_PARAM_USER_DATA,
@@ -290,15 +308,11 @@
         },
         stream_renderer_param{
             .key = STREAM_RENDERER_PARAM_RENDERER_FLAGS,
-            .value =
-                static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_SURFACELESS_BIT) |
-                (withGl ? static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_EGL_BIT |
-                                                STREAM_RENDERER_FLAGS_USE_GLES_BIT)
-                        : 0) |
-                (withVk ? static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_VK_BIT) : 0) |
-                (withVkSnapshots ? static_cast<uint64_t>(STREAM_RENDERER_FLAGS_VULKAN_SNAPSHOTS)
-                                 : 0),
-        },
+            .value = static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_SURFACELESS_BIT) |
+                     (withGl ? static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_EGL_BIT |
+                                                     STREAM_RENDERER_FLAGS_USE_GLES_BIT)
+                             : 0) |
+                     (withVk ? static_cast<uint64_t>(STREAM_RENDERER_FLAGS_USE_VK_BIT) : 0)},
         stream_renderer_param{
             .key = STREAM_RENDERER_PARAM_WIN0_WIDTH,
             .value = 32,
@@ -581,6 +595,25 @@
     EnqueueVirtioGpuTask(contextId, std::move(detachTask));
 }
 
+void EmulatedVirtioGpu::EmulatedVirtioGpuImpl::SnapshotSave(std::string directory) {
+    uint32_t contextId = 0;
+
+    VirtioGpuTaskSnapshotSave saveTask{
+        .directory = directory,
+    };
+    EnqueueVirtioGpuTask(contextId, std::move(saveTask));
+}
+
+void EmulatedVirtioGpu::EmulatedVirtioGpuImpl::SnapshotRestore(std::string directory) {
+    uint32_t contextId = 0;
+
+    VirtioGpuTaskSnapshotRestore restoreTask{
+        .directory = directory,
+    };
+
+    EnqueueVirtioGpuTask(contextId, std::move(restoreTask));
+}
+
 int EmulatedVirtioGpu::EmulatedVirtioGpuImpl::ExecBuffer(uint32_t contextId,
                                                          struct VirtGpuExecBuffer& execbuffer,
                                                          std::optional<uint32_t> blobResourceId) {
@@ -865,6 +898,14 @@
     stream_renderer_resource_unref(task.resourceId);
 }
 
+void EmulatedVirtioGpu::EmulatedVirtioGpuImpl::DoTask(VirtioGpuTaskSnapshotSave task) {
+    stream_renderer_snapshot(task.directory.c_str());
+}
+
+void EmulatedVirtioGpu::EmulatedVirtioGpuImpl::DoTask(VirtioGpuTaskSnapshotRestore task) {
+    stream_renderer_restore(task.directory.c_str());
+}
+
 void EmulatedVirtioGpu::EmulatedVirtioGpuImpl::DoTask(VirtioGpuTaskWithWaitable task) {
     std::visit(
         [this](auto&& work) {
@@ -891,6 +932,10 @@
                 DoTask(std::move(work));
             } else if constexpr (std::is_same_v<T, VirtioGpuTaskUnrefResource>) {
                 DoTask(std::move(work));
+            } else if constexpr (std::is_same_v<T, VirtioGpuTaskSnapshotSave>) {
+                DoTask(std::move(work));
+            } else if constexpr (std::is_same_v<T, VirtioGpuTaskSnapshotRestore>) {
+                DoTask(std::move(work));
             }
         },
         task.task);
@@ -1032,6 +1077,12 @@
     mImpl->DestroyResource(contextId, resourceId);
 }
 
+void EmulatedVirtioGpu::SnapshotSave(std::string directory) { mImpl->SnapshotSave(directory); }
+
+void EmulatedVirtioGpu::SnapshotRestore(std::string directory) {
+    mImpl->SnapshotRestore(directory);
+}
+
 int EmulatedVirtioGpu::WaitOnEmulatedFence(int fenceAsFileDescriptor, int timeoutMilliseconds) {
     return mImpl->WaitOnEmulatedFence(fenceAsFileDescriptor, timeoutMilliseconds);
 }
diff --git a/guest/platform/rutabaga/RutabagaLayer.h b/guest/platform/rutabaga/RutabagaLayer.h
index 7c6443b..e7a8073 100644
--- a/guest/platform/rutabaga/RutabagaLayer.h
+++ b/guest/platform/rutabaga/RutabagaLayer.h
@@ -63,6 +63,9 @@
 
    int WaitOnEmulatedFence(int fenceAsFileDescriptor, int timeoutMilliseconds);
 
+   void SnapshotSave(std::string directory);
+   void SnapshotRestore(std::string directory);
+
   private:
     EmulatedVirtioGpu();
 
diff --git a/host/include/gfxstream/virtio-gpu-gfxstream-renderer-unstable.h b/host/include/gfxstream/virtio-gpu-gfxstream-renderer-unstable.h
index 429bfa5..952718e 100644
--- a/host/include/gfxstream/virtio-gpu-gfxstream-renderer-unstable.h
+++ b/host/include/gfxstream/virtio-gpu-gfxstream-renderer-unstable.h
@@ -102,6 +102,10 @@
 VG_EXPORT int stream_renderer_resource_get_info(int res_handle,
                                                 struct stream_renderer_resource_info* info);
 
+VG_EXPORT int stream_renderer_snapshot(const char* dir);
+
+VG_EXPORT int stream_renderer_restore(const char* dir);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/host/include/gfxstream/virtio-gpu-gfxstream-renderer.h b/host/include/gfxstream/virtio-gpu-gfxstream-renderer.h
index 20383d3..081befc 100644
--- a/host/include/gfxstream/virtio-gpu-gfxstream-renderer.h
+++ b/host/include/gfxstream/virtio-gpu-gfxstream-renderer.h
@@ -252,7 +252,6 @@
     STREAM_RENDERER_FLAGS_USE_EXTERNAL_BLOB = 1 << 6,
     STREAM_RENDERER_FLAGS_USE_SYSTEM_BLOB = 1 << 7,
     STREAM_RENDERER_FLAGS_VULKAN_NATIVE_SWAPCHAIN_BIT = 1 << 8,
-    STREAM_RENDERER_FLAGS_VULKAN_SNAPSHOTS = 1 << 9,
 };
 
 #endif
diff --git a/host/meson.build b/host/meson.build
index c37eac4..060bbf7 100644
--- a/host/meson.build
+++ b/host/meson.build
@@ -23,6 +23,9 @@
   endif
 endif
 
+pkg_cflags = []
+pkg_cflags += '-DGFXSTREAM_UNSTABLE=1'
+
 #===============#
 # Dependencies  #
 #===============#
@@ -280,5 +283,6 @@
 pkg = import('pkgconfig')
 pkg.generate(gfxstream_backend,
              description: 'gfxstream backend',
+             extra_cflags: pkg_cflags,
              subdirs: 'gfxstream'
 )
diff --git a/host/virtio-gpu-gfxstream-renderer.cpp b/host/virtio-gpu-gfxstream-renderer.cpp
index 2d8acd1..1d9706d 100644
--- a/host/virtio-gpu-gfxstream-renderer.cpp
+++ b/host/virtio-gpu-gfxstream-renderer.cpp
@@ -45,6 +45,10 @@
 #include "virtgpu_gfxstream_protocol.h"
 #include "vk_util.h"
 
+#ifdef GFXSTREAM_ENABLE_HOST_VK_SNAPSHOT
+#include "aemu/base/files/StdioStream.h"
+#endif
+
 extern "C" {
 #include "drm_fourcc.h"
 #include "gfxstream/virtio-gpu-gfxstream-renderer-unstable.h"
@@ -1977,6 +1981,52 @@
     return sRenderer()->vulkanInfo(res_handle, vulkan_info);
 }
 
+VG_EXPORT int stream_renderer_snapshot(const char* dir) {
+#ifdef GFXSTREAM_ENABLE_HOST_VK_SNAPSHOT
+    std::string dirString(dir);
+
+    std::string snapshotFileName = dirString + "snapshot.bin";
+
+    std::unique_ptr<android::base::StdioStream> stream(new android::base::StdioStream(
+        fopen(snapshotFileName.c_str(), "wb"), android::base::StdioStream::kOwner));
+
+    android_getOpenglesRenderer()->pauseAllPreSave();
+    android::snapshot::SnapshotSaveStream saveStream{
+        .stream = stream.get(),
+    };
+
+    android_getOpenglesRenderer()->save(saveStream.stream, saveStream.textureSaver);
+    return 0;
+#else
+    stream_renderer_error("Snapshot save requested without support.");
+    return -EINVAL;
+#endif
+}
+
+VG_EXPORT int stream_renderer_restore(const char* dir) {
+#ifdef GFXSTREAM_ENABLE_HOST_VK_SNAPSHOT
+    std::string dirString(dir);
+    std::string snapshotFileName = dirString + "snapshot.bin";
+
+    std::unique_ptr<android::base::StdioStream> stream(new android::base::StdioStream(
+        fopen(snapshotFileName.c_str(), "rb"), android::base::StdioStream::kOwner));
+
+    android::snapshot::SnapshotLoadStream loadStream{
+        .stream = stream.get(),
+    };
+
+    android_getOpenglesRenderer()->load(loadStream.stream, loadStream.textureLoader);
+
+    // In end2end tests, we don't really do snapshot save for render threads.
+    // We will need to resume all render threads without waiting for snapshot.
+    android_getOpenglesRenderer()->resumeAll(false);
+    return 0;
+#else
+    stream_renderer_error("Snapshot save requested without support.");
+    return -EINVAL;
+#endif
+}
+
 static const GoldfishPipeServiceOps goldfish_pipe_service_ops = {
     // guest_open()
     [](GoldfishHwPipe* hwPipe) -> GoldfishHostPipe* {
@@ -2108,7 +2158,6 @@
     android::base::setEnvironmentVariable("ANDROID_EMU_HEADLESS", "1");
     bool enableVk = (renderer_flags & STREAM_RENDERER_FLAGS_USE_VK_BIT);
     bool enableGles = (renderer_flags & STREAM_RENDERER_FLAGS_USE_GLES_BIT);
-    bool enableVkSnapshot = (renderer_flags & STREAM_RENDERER_FLAGS_VULKAN_SNAPSHOTS);
 
     bool egl2eglByEnv = android::base::getEnvironmentVariable("ANDROID_EGL_ON_EGL") == "1";
     bool egl2eglByFlag = renderer_flags & STREAM_RENDERER_FLAGS_USE_EGL_BIT;
@@ -2157,7 +2206,6 @@
     feature_set_enabled_override(kFeature_NativeTextureDecompression, false);
     feature_set_enabled_override(kFeature_GLDirectMem, false);
     feature_set_enabled_override(kFeature_Vulkan, enableVk);
-    feature_set_enabled_override(kFeature_VulkanSnapshots, enableVkSnapshot);
     feature_set_enabled_override(kFeature_VulkanNullOptionalStrings, true);
     feature_set_enabled_override(kFeature_VulkanShaderFloat16Int8, true);
     feature_set_enabled_override(kFeature_HostComposition, true);
@@ -2172,6 +2220,10 @@
     feature_set_enabled_override(kFeature_ExternalBlob, useExternalBlob);
     feature_set_enabled_override(kFeature_SystemBlob, useSystemBlob);
 
+    if (android::base::getEnvironmentVariable("ANDROID_GFXSTREAM_CAPTURE_VK_SNAPSHOT") == "1") {
+        feature_set_enabled_override(kFeature_VulkanSnapshots, true);
+    }
+
     android::featurecontrol::productFeatureOverride();
 
     if (useVulkanNativeSwapchain && !enableVk) {