gl: Support ColorBuffers of A2B10G10R10 format.

There's no equivalent internal formats to Vulkan A2B10G10R10 format
in GL, which makes it hard to create a color buffer with such
format.

In this change, we introduce an enum GL_BGR10_A2_ANGLEX to represent
A2B10G10R10 formats and translate the format to corresponding
format/type when calling glTexImage2D to create textures.

Change-Id: I63274ae4e75c678e07f0ef0df8ea4f0729d2cddf
diff --git a/stream-servers/ColorBuffer.cpp b/stream-servers/ColorBuffer.cpp
index 24d17ae..b36c871 100644
--- a/stream-servers/ColorBuffer.cpp
+++ b/stream-servers/ColorBuffer.cpp
@@ -91,21 +91,28 @@
         case GL_RGB10_A2:
         case GL_RGBA16F:
             return GL_RGBA;
+        case GL_BGRA8_EXT:
+        case GL_BGR10_A2_ANGLEX:
+            return GL_BGRA_EXT;
         default: // already unsized
             return format;
     }
 }
 
-static bool sGetFormatParameters(
-    GLint internalFormat,
-    GLenum* texFormat,
-    GLenum* pixelType,
-    int* bytesPerPixel,
-    GLint* sizedInternalFormat,
-    bool* isBlob) {
+static bool sGetFormatParameters(GLint* internalFormat,
+                                 GLenum* texFormat,
+                                 GLenum* pixelType,
+                                 int* bytesPerPixel,
+                                 GLint* sizedInternalFormat,
+                                 bool* isBlob) {
+    if (!internalFormat) {
+        fprintf(stderr, "%s: error: internal format not provided\n", __func__);
+        return false;
+    }
+
     *isBlob = false;
 
-    switch (internalFormat) {
+    switch (*internalFormat) {
         case GL_RGB:
         case GL_RGB8:
             *texFormat = GL_RGB;
@@ -165,6 +172,15 @@
             *bytesPerPixel = 4;
             *sizedInternalFormat = GL_BGRA8_EXT;
             return true;
+        case GL_BGR10_A2_ANGLEX:
+            *texFormat = GL_RGBA;
+            *pixelType = GL_UNSIGNED_INT_2_10_10_10_REV;
+            *bytesPerPixel = 4;
+            *internalFormat = GL_RGB10_A2_EXT;
+            // GL_BGR10_A2_ANGLEX is actually not a valid GL format. We should
+            // replace it with a normal GL internal format instead.
+            *sizedInternalFormat = GL_BGR10_A2_ANGLEX;
+            return true;
         case GL_R8:
         case GL_RED:
             *texFormat = GL_RED;
@@ -180,8 +196,8 @@
             *sizedInternalFormat = GL_RG8;
             return true;
         default:
-            fprintf(stderr, "%s: Unknown format 0x%x\n",
-                    __func__, internalFormat);
+            fprintf(stderr, "%s: Unknown format 0x%x\n", __func__,
+                    *internalFormat);
             return false;
     }
 }
@@ -190,7 +206,7 @@
 ColorBuffer* ColorBuffer::create(EGLDisplay p_display,
                                  int p_width,
                                  int p_height,
-                                 GLenum p_internalFormat,
+                                 GLint p_internalFormat,
                                  FrameworkFormat p_frameworkFormat,
                                  HandleType hndl,
                                  Helper* helper,
@@ -201,11 +217,9 @@
     GLint p_sizedInternalFormat = GL_RGBA8;
     bool isBlob = false;;
 
-    if (!sGetFormatParameters(
-            p_internalFormat,
-            &texFormat, &pixelType, &bytesPerPixel,
-            &p_sizedInternalFormat,
-            &isBlob)) {
+    if (!sGetFormatParameters(&p_internalFormat, &texFormat, &pixelType,
+                              &bytesPerPixel, &p_sizedInternalFormat,
+                              &isBlob)) {
         fprintf(stderr, "ColorBuffer::create invalid format 0x%x\n",
                 p_internalFormat);
         return NULL;
@@ -235,6 +249,12 @@
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    // Swizzle B/R channel for BGR10_A2 images.
+    if (p_sizedInternalFormat == GL_BGR10_A2_ANGLEX) {
+        s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
+        s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
+        cb->m_BRSwizzle = true;
+    }
 
     //
     // create another texture for that colorbuffer for blit
@@ -248,6 +268,12 @@
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    // Swizzle B/R channel for BGR10_A2 images.
+    if (p_sizedInternalFormat == GL_BGR10_A2_ANGLEX) {
+        s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
+        s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
+        cb->m_BRSwizzle = true;
+    }
 
     cb->m_width = p_width;
     cb->m_height = p_height;
@@ -408,7 +434,8 @@
     GLint sizedInternalFormat = GL_RGBA8;
     int bpp = 4;
     bool isBlob = false;
-    if (!sGetFormatParameters(internalformat, &texFormat, &pixelType, &bpp, &sizedInternalFormat, &isBlob)) {
+    if (!sGetFormatParameters(&internalformat, &texFormat, &pixelType, &bpp,
+                              &sizedInternalFormat, &isBlob)) {
         fprintf(stderr, "%s: WARNING: reformat failed. internal format: 0x%x\n",
                 __func__, internalformat);
     }
@@ -978,8 +1005,13 @@
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 
-    if (m_sizedInternalFormat == GL_BGRA8_EXT) {
-        s_gles2.glTexStorageMem2DEXT(GL_TEXTURE_2D, 1, GL_RGBA8, m_width, m_height, m_memoryObject, 0);
+    if (m_sizedInternalFormat == GL_BGRA8_EXT ||
+        m_sizedInternalFormat == GL_BGR10_A2_ANGLEX) {
+        GLint internalFormat = m_sizedInternalFormat == GL_BGRA8_EXT
+                                       ? GL_RGBA8
+                                       : GL_RGB10_A2_EXT;
+        s_gles2.glTexStorageMem2DEXT(GL_TEXTURE_2D, 1, internalFormat, m_width,
+                                     m_height, m_memoryObject, 0);
         s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
         s_gles2.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
         m_BRSwizzle = true;
diff --git a/stream-servers/ColorBuffer.h b/stream-servers/ColorBuffer.h
index 9d4eb91..8153004 100644
--- a/stream-servers/ColorBuffer.h
+++ b/stream-servers/ColorBuffer.h
@@ -30,6 +30,9 @@
 #include "RenderContext.h"
 #include "snapshot/LazySnapshotObj.h"
 
+// From ANGLE "src/common/angleutils.h"
+#define GL_BGR10_A2_ANGLEX 0x6AF9
+
 class TextureDraw;
 class TextureResize;
 class YUVConverter;
@@ -134,7 +137,7 @@
     static ColorBuffer* create(EGLDisplay p_display,
                                int p_width,
                                int p_height,
-                               GLenum p_internalFormat,
+                               GLint p_internalFormat,
                                FrameworkFormat p_frameworkFormat,
                                HandleType hndl,
                                Helper* helper,
@@ -286,8 +289,8 @@
     GLuint m_width = 0;
     GLuint m_height = 0;
     GLuint m_fbo = 0;
-    GLenum m_internalFormat = 0;
-    GLenum m_sizedInternalFormat = 0;
+    GLint m_internalFormat = 0;
+    GLint m_sizedInternalFormat = 0;
 
     // |m_format| and |m_type| are for reformatting purposes only
     // to work around bugs in the guest. No need to snapshot those.
diff --git a/stream-servers/vulkan/VkCommonOperations.cpp b/stream-servers/vulkan/VkCommonOperations.cpp
index 1c2407d..99b3793 100644
--- a/stream-servers/vulkan/VkCommonOperations.cpp
+++ b/stream-servers/vulkan/VkCommonOperations.cpp
@@ -1316,6 +1316,8 @@
         case GL_RGB10_A2:
         case GL_UNSIGNED_INT_10_10_10_2_OES:
             return VK_FORMAT_A2R10G10B10_UNORM_PACK32;
+        case GL_BGR10_A2_ANGLEX:
+            return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
         case GL_RGBA16F:
             return VK_FORMAT_R16G16B16A16_SFLOAT;
         case GL_BGRA_EXT: