Revert "Revert "Instrument tests for glRenderer""
This reverts commit 3b0e0437f125a477304b6a937eb6d2ed5b307d7a.
Reason for revert: Fix test and roll forward
The failure is caused by the GL thread check. We can't reproduce this locally or with ACID. In this CL, we post all the gl operations on an executor and see if it makes the issue go away. The fix is not verified because we can't reproduce the failure.
Bug: 295407763
Test: manual test and tested with ACID device
Change-Id: I85ed7b9d49a2feef118460dfc2f864e2947a1a83
diff --git a/camera/camera-effects/build.gradle b/camera/camera-effects/build.gradle
index 6c618ee..26dc06e 100644
--- a/camera/camera-effects/build.gradle
+++ b/camera/camera-effects/build.gradle
@@ -33,6 +33,13 @@
androidTestImplementation(libs.testRunner)
androidTestImplementation(libs.testRules)
androidTestImplementation(libs.truth)
+ androidTestImplementation(project(":camera:camera-testing")) {
+ // Ensure camera-testing does not pull in androidx.test dependencies
+ exclude(group:"androidx.test")
+ }
+ androidTestImplementation(libs.kotlinStdlib)
+ androidTestImplementation(libs.kotlinCoroutinesAndroid)
+ androidTestImplementation("androidx.concurrent:concurrent-futures-ktx:1.1.0")
}
android {
defaultConfig {
diff --git a/camera/camera-effects/src/androidTest/java/androidx/camera/effects/opengl/GlRendererDeviceTest.kt b/camera/camera-effects/src/androidTest/java/androidx/camera/effects/opengl/GlRendererDeviceTest.kt
index 795082f..73a5942 100644
--- a/camera/camera-effects/src/androidTest/java/androidx/camera/effects/opengl/GlRendererDeviceTest.kt
+++ b/camera/camera-effects/src/androidTest/java/androidx/camera/effects/opengl/GlRendererDeviceTest.kt
@@ -16,11 +16,33 @@
package androidx.camera.effects.opengl
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.graphics.Color
+import android.graphics.Paint
+import android.graphics.PorterDuff
+import android.graphics.Rect
+import android.graphics.SurfaceTexture
+import android.opengl.Matrix
+import android.os.Handler
+import android.os.Looper
+import android.util.Size
+import android.view.Surface
+import androidx.camera.testing.impl.TestImageUtil.createBitmap
+import androidx.camera.testing.impl.TestImageUtil.getAverageDiff
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.SdkSuppress
import androidx.test.filters.SmallTest
import com.google.common.truth.Truth.assertThat
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeoutOrNull
import org.junit.After
+import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
@@ -33,21 +55,172 @@
@SdkSuppress(minSdkVersion = 21)
class GlRendererDeviceTest {
+ companion object {
+ private const val WIDTH = 640
+ private const val HEIGHT = 480
+ private const val TIMESTAMP_NS = 0L
+ }
+
+ private val input = createBitmap(WIDTH, HEIGHT)
+ private val overlay = createOverlayBitmap()
+ private val transparentOverlay = createTransparentOverlay()
+
private val glRenderer = GlRenderer()
+ private lateinit var inputSurface: Surface
+ private lateinit var inputTexture: SurfaceTexture
+ private lateinit var inputExecutor: ExecutorService
+
+ private lateinit var outputSurface: Surface
+ private lateinit var outputTexture: SurfaceTexture
+
+ private val identityMatrix = FloatArray(16).apply {
+ Matrix.setIdentityM(this, 0)
+ }
@Before
- fun setUp() {
- glRenderer.init()
+ fun setUp() = runBlocking {
+ inputExecutor = Executors.newSingleThreadExecutor()
+ withContext(inputExecutor.asCoroutineDispatcher()) {
+ glRenderer.init()
+ inputTexture = SurfaceTexture(glRenderer.inputTextureId).apply {
+ setDefaultBufferSize(WIDTH, HEIGHT)
+ }
+ inputSurface = Surface(inputTexture)
+ }
+ outputTexture = SurfaceTexture(0).apply {
+ setDefaultBufferSize(WIDTH, HEIGHT)
+ }
+ outputSurface = Surface(outputTexture)
}
@After
fun tearDown() {
- glRenderer.release()
+ inputExecutor.execute {
+ glRenderer.release()
+ inputTexture.release()
+ inputSurface.release()
+ }
+ outputTexture.release()
+ outputSurface.release()
+ inputExecutor.shutdown()
}
- // TODO(b/295407763): verify the input/output of the OpenGL renderer
+ @Test(expected = IllegalStateException::class)
+ fun renderInputWhenUninitialized_throwsException() {
+ val glRenderer = GlRenderer()
+ try {
+ glRenderer.renderInputToSurface(TIMESTAMP_NS, identityMatrix, outputSurface)
+ } finally {
+ glRenderer.release()
+ }
+ }
+
@Test
- fun placeholder() {
- assertThat(true).isTrue()
+ fun drawInputToQueue_snapshot() = runBlocking(inputExecutor.asCoroutineDispatcher()) {
+ // Arrange: upload a overlay and create a texture queue.
+ glRenderer.uploadOverlay(overlay)
+ drawInputSurface(input)
+ val queue = glRenderer.createBufferTextureIds(1, Size(WIDTH, HEIGHT))
+ // Act: draw input to the queue and then to the output.
+ glRenderer.renderInputToQueueTexture(queue[0])
+ val bitmap =
+ glRenderer.renderQueueTextureToBitmap(queue[0], WIDTH, HEIGHT, identityMatrix)
+ // Assert: the output is the input with overlay.
+ assertOverlayColor(bitmap)
+ }
+
+ @Test
+ fun drawInputWithoutOverlay_snapshot() = runBlocking(inputExecutor.asCoroutineDispatcher()) {
+ // Arrange: upload a transparent overlay.
+ glRenderer.uploadOverlay(transparentOverlay)
+ drawInputSurface(input)
+ // Act.
+ val output = glRenderer.renderInputToBitmap(WIDTH, HEIGHT, identityMatrix)
+ // Assert: the output is the same as the input.
+ assertThat(getAverageDiff(output, input)).isEqualTo(0)
+ }
+
+ /**
+ * Tests that the input is rendered to the output surface with the overlay.
+ */
+ private fun assertOverlayColor(bitmap: Bitmap) {
+ // Top left quadrant is white.
+ assertThat(
+ getAverageDiff(
+ bitmap,
+ Rect(0, 0, WIDTH / 2, HEIGHT / 2),
+ Color.WHITE
+ )
+ ).isEqualTo(0)
+ assertThat(
+ getAverageDiff(
+ bitmap,
+ Rect(WIDTH / 2, 0, WIDTH, HEIGHT / 2),
+ Color.GREEN
+ )
+ ).isEqualTo(0)
+ assertThat(
+ getAverageDiff(
+ bitmap,
+ Rect(WIDTH / 2, HEIGHT / 2, WIDTH, HEIGHT),
+ Color.YELLOW
+ )
+ ).isEqualTo(0)
+ assertThat(
+ getAverageDiff(
+ bitmap,
+ Rect(0, HEIGHT / 2, WIDTH / 2, HEIGHT),
+ Color.BLUE
+ )
+ ).isEqualTo(0)
+ }
+
+ /**
+ * Draws the bitmap to the input surface and waits for the frame to be available.
+ */
+ private suspend fun drawInputSurface(bitmap: Bitmap) {
+ val deferredOnFrameAvailable = CompletableDeferred<Unit>()
+ inputTexture.setOnFrameAvailableListener({
+ deferredOnFrameAvailable.complete(Unit)
+ }, Handler(Looper.getMainLooper()))
+
+ // Draw bitmap to inputSurface.
+ val canvas = inputSurface.lockCanvas(null)
+ canvas.drawBitmap(bitmap, 0f, 0f, null)
+ inputSurface.unlockCanvasAndPost(canvas)
+
+ // Wait for frame available and update texture.
+ withTimeoutOrNull(5_000) {
+ deferredOnFrameAvailable.await()
+ } ?: Assert.fail("Timed out waiting for SurfaceTexture frame available.")
+ inputTexture.updateTexImage()
+ }
+
+ /**
+ * Creates a bitmap with a white top-left quadrant.
+ */
+ private fun createOverlayBitmap(): Bitmap {
+ val bitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888)
+ val centerX = (WIDTH / 2).toFloat()
+ val centerY = (HEIGHT / 2).toFloat()
+
+ val canvas = Canvas(bitmap)
+ canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
+
+ val paint = Paint()
+ paint.style = Paint.Style.FILL
+ paint.color = Color.WHITE
+ canvas.drawRect(0f, 0f, centerX, centerY, paint)
+ return bitmap
+ }
+
+ /**
+ * Creates a transparent bitmap.
+ */
+ private fun createTransparentOverlay(): Bitmap {
+ val bitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888)
+ val canvas = Canvas(bitmap)
+ canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
+ return bitmap
}
}
diff --git a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramCopy.java b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramCopy.java
index 5114b56..2336fa3 100644
--- a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramCopy.java
+++ b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramCopy.java
@@ -17,6 +17,7 @@
package androidx.camera.effects.opengl;
import static androidx.camera.effects.opengl.Utils.checkGlErrorOrThrow;
+import static androidx.camera.effects.opengl.Utils.createFbo;
import static androidx.camera.effects.opengl.Utils.drawArrays;
import android.opengl.GLES11Ext;
@@ -61,10 +62,7 @@
protected void configure() {
super.configure();
// Create a FBO for attaching the output texture.
- int[] fbos = new int[1];
- GLES20.glGenFramebuffers(1, fbos, 0);
- checkGlErrorOrThrow("glGenFramebuffers");
- mFbo = fbos[0];
+ mFbo = createFbo();
}
@Override
diff --git a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramOverlay.java b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramOverlay.java
index 8bea7e0..61a1a9f 100644
--- a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramOverlay.java
+++ b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlProgramOverlay.java
@@ -16,9 +16,16 @@
package androidx.camera.effects.opengl;
+import static androidx.camera.core.ImageProcessingUtil.copyByteBufferToBitmap;
import static androidx.camera.effects.opengl.Utils.checkGlErrorOrThrow;
import static androidx.camera.effects.opengl.Utils.checkLocationOrThrow;
+import static androidx.camera.effects.opengl.Utils.configureTexture2D;
+import static androidx.camera.effects.opengl.Utils.createFbo;
+import static androidx.camera.effects.opengl.Utils.createTextureId;
+import static androidx.camera.effects.opengl.Utils.drawArrays;
+import static androidx.core.util.Preconditions.checkArgument;
+import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.view.Surface;
@@ -26,6 +33,8 @@
import androidx.annotation.RequiresApi;
import androidx.camera.core.Logger;
+import java.nio.ByteBuffer;
+
/**
* A GL program that copies the source while overlaying a texture on top of it.
*/
@@ -34,6 +43,8 @@
private static final String TAG = "GlProgramOverlay";
+ private static final int SNAPSHOT_PIXEL_STRIDE = 4;
+
static final String TEXTURE_MATRIX = "uTexMatrix";
static final String OVERLAY_SAMPLER = "samplerOverlayTexture";
@@ -109,7 +120,91 @@
@NonNull float[] matrix, @NonNull GlContext glContext, @NonNull Surface surface,
long timestampNs) {
use();
+ uploadParameters(inputTextureTarget, inputTextureId, overlayTextureId, matrix);
+ try {
+ glContext.drawAndSwap(surface, timestampNs);
+ } catch (IllegalStateException e) {
+ Logger.w(TAG, "Failed to draw the frame", e);
+ }
+ }
+ /**
+ * Draws the input texture and overlay to a Bitmap.
+ *
+ * @param inputTextureTarget the texture target of the input texture. This could be either
+ * GLES11Ext.GL_TEXTURE_EXTERNAL_OES or GLES20.GL_TEXTURE_2D,
+ * depending if copying from an external texture or a 2D texture.
+ * @param inputTextureId the texture id of the input texture. This could be either an
+ * external texture or a 2D texture.
+ * @param overlayTextureId the texture id of the overlay texture. This must be a 2D texture.
+ * @param width the width of the output bitmap.
+ * @param height the height of the output bitmap.
+ * @param matrix the texture transformation matrix.
+ */
+ @NonNull
+ Bitmap snapshot(int inputTextureTarget, int inputTextureId, int overlayTextureId, int width,
+ int height, @NonNull float[] matrix) {
+ use();
+ // Allocate buffer.
+ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * SNAPSHOT_PIXEL_STRIDE);
+ // Take a snapshot.
+ snapshot(inputTextureTarget, inputTextureId, overlayTextureId, width, height,
+ matrix, byteBuffer);
+ // Create a Bitmap and copy the bytes over.
+ Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+ byteBuffer.rewind();
+ copyByteBufferToBitmap(bitmap, byteBuffer, width * SNAPSHOT_PIXEL_STRIDE);
+ return bitmap;
+ }
+
+ /**
+ * Draws the input texture and overlay to a FBO and download the bytes to the given ByteBuffer.
+ */
+ private void snapshot(int inputTextureTarget,
+ int inputTextureId, int overlayTextureId, int width,
+ int height, @NonNull float[] textureTransform, @NonNull ByteBuffer byteBuffer) {
+ checkArgument(byteBuffer.capacity() == width * height * 4,
+ "ByteBuffer capacity is not equal to width * height * 4.");
+ checkArgument(byteBuffer.isDirect(), "ByteBuffer is not direct.");
+
+ // Create a FBO as the drawing target.
+ int fbo = createFbo();
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo);
+ checkGlErrorOrThrow("glBindFramebuffer");
+ // Create the texture behind the FBO
+ int textureId = createTextureId();
+ configureTexture2D(textureId);
+ GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGB, width,
+ height, 0, GLES20.GL_RGB, GLES20.GL_UNSIGNED_BYTE, null);
+ checkGlErrorOrThrow("glTexImage2D");
+ // Attach the texture to the FBO
+ GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
+ GLES20.GL_TEXTURE_2D, textureId, 0);
+ checkGlErrorOrThrow("glFramebufferTexture2D");
+
+ // Draw
+ uploadParameters(inputTextureTarget, inputTextureId, overlayTextureId, textureTransform);
+ drawArrays(width, height);
+
+ // Download the pixels from the FBO
+ GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE,
+ byteBuffer);
+ checkGlErrorOrThrow("glReadPixels");
+
+ // Clean up
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
+ checkGlErrorOrThrow("glBindFramebuffer");
+ GLES20.glDeleteTextures(1, new int[]{textureId}, 0);
+ checkGlErrorOrThrow("glDeleteTextures");
+ GLES20.glDeleteFramebuffers(1, new int[]{fbo}, 0);
+ checkGlErrorOrThrow("glDeleteFramebuffers");
+ }
+
+ /**
+ * Uploads the parameters to the shader.
+ */
+ private void uploadParameters(int inputTextureTarget, int inputTextureId, int overlayTextureId,
+ @NonNull float[] matrix) {
// Uploads the texture transformation matrix.
GLES20.glUniformMatrix4fv(mTextureMatrixLoc, 1, false, matrix, 0);
checkGlErrorOrThrow("glUniformMatrix4fv");
@@ -123,11 +218,5 @@
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, overlayTextureId);
checkGlErrorOrThrow("glBindTexture");
-
- try {
- glContext.drawAndSwap(surface, timestampNs);
- } catch (IllegalStateException e) {
- Logger.w(TAG, "Failed to draw the frame", e);
- }
}
}
diff --git a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlRenderer.java b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlRenderer.java
index 50488dc..8046e43f 100644
--- a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlRenderer.java
+++ b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/GlRenderer.java
@@ -235,6 +235,7 @@
*/
public void renderQueueTextureToSurface(int textureId, long timestampNs,
@NonNull float[] textureTransform, @NonNull Surface surface) {
+ checkGlThreadAndInitialized();
mGlProgramOverlay.draw(GLES20.GL_TEXTURE_2D, textureId, mOverlayTextureId,
textureTransform, mGlContext, surface, timestampNs);
}
@@ -245,9 +246,31 @@
* <p>The texture ID must be from the latest return value of{@link #createBufferTextureIds}.
*/
public void renderInputToQueueTexture(int textureId) {
+ checkGlThreadAndInitialized();
mGlProgramCopy.draw(mInputTextureId, textureId, mQueueTextureWidth, mQueueTextureHeight);
}
+ /**
+ * Renders a queued texture to a Bitmap and returns.
+ */
+ @NonNull
+ public Bitmap renderQueueTextureToBitmap(int textureId, int width, int height,
+ @NonNull float[] textureTransform) {
+ checkGlThreadAndInitialized();
+ return mGlProgramOverlay.snapshot(GLES20.GL_TEXTURE_2D, textureId, mOverlayTextureId,
+ width, height, textureTransform);
+ }
+
+ /**
+ * Renders the input texture to a Bitmap and returns.
+ */
+ @NonNull
+ public Bitmap renderInputToBitmap(int width, int height, @NonNull float[] textureTransform) {
+ checkGlThreadAndInitialized();
+ return mGlProgramOverlay.snapshot(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mInputTextureId,
+ mOverlayTextureId, width, height, textureTransform);
+ }
+
// --- Private methods ---
private void checkGlThreadAndInitialized() {
diff --git a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/Utils.java b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/Utils.java
index e8f2d49..7c28bb1 100644
--- a/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/Utils.java
+++ b/camera/camera-effects/src/main/java/androidx/camera/effects/opengl/Utils.java
@@ -100,6 +100,16 @@
}
/**
+ * Creates a single FBO.
+ */
+ static int createFbo() {
+ int[] fbos = new int[1];
+ GLES20.glGenFramebuffers(1, fbos, 0);
+ checkGlErrorOrThrow("glGenFramebuffers");
+ return fbos[0];
+ }
+
+ /**
* Configures the texture as a 2D texture.
*/
static void configureTexture2D(int textureId) {
