Trace-based FrameTimingMetric, Step 2
Bug: 195987329
Bug: 183129298
Fixes: 200082324
Test: TrivialListScrollBenchmark # Validated on 29 & 31
Test: ./gradlew benchmark:benchmark-common:cC benchmark:benchmark-macro:cC
Completes migration of FrameTimingMetric to be trace-based, and
surface FrameNegativeSlack when available.
This change also makes several related changes to frame metrics coming
from FrameTimingQuery:
* Deleted FrameTime, as it's always preferable to use something
deadline based
* Renamed BasicFrameTime -> FrameCpuTime for clarity, and to match metric plan
* Made FrameSlack negative, so the *worst* frames show up in 99P,
instead of best. This avoids having to differentate between low=bad
and high=bad metrics.
Adds FrameTimingGfxInfoMetric to existing scroll benchmarks for
temporary continuity in CI tracking.
Change-Id: Ied56278b1f76f37df44181dc736590753aa1cb30
diff --git a/benchmark/benchmark-macro/api/restricted_current.txt b/benchmark/benchmark-macro/api/restricted_current.txt
index ab915af..fd0ce9c 100644
--- a/benchmark/benchmark-macro/api/restricted_current.txt
+++ b/benchmark/benchmark-macro/api/restricted_current.txt
@@ -32,12 +32,12 @@
method @RestrictTo(androidx.annotation.RestrictTo.Scope.LIBRARY_GROUP_PREFIX) public static boolean isSupportedWithVmSettings(androidx.benchmark.macro.CompilationMode);
}
- public final class FrameTimingMetric extends androidx.benchmark.macro.Metric {
- ctor public FrameTimingMetric();
+ @RestrictTo(androidx.annotation.RestrictTo.Scope.LIBRARY_GROUP_PREFIX) public final class FrameTimingGfxInfoMetric extends androidx.benchmark.macro.Metric {
+ ctor public FrameTimingGfxInfoMetric();
}
- @RestrictTo(androidx.annotation.RestrictTo.Scope.LIBRARY_GROUP_PREFIX) public final class FrameTimingTraceMetric extends androidx.benchmark.macro.Metric {
- ctor public FrameTimingTraceMetric();
+ public final class FrameTimingMetric extends androidx.benchmark.macro.Metric {
+ ctor public FrameTimingMetric();
}
public final class IdeSummaryStringKt {
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/IdeSummaryStringTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/IdeSummaryStringTest.kt
index 3357a1f..a150eee 100644
--- a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/IdeSummaryStringTest.kt
+++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/IdeSummaryStringTest.kt
@@ -123,7 +123,7 @@
assertEquals(
"""
|foo
- | Metric1 P50 50.0, P90 90.0, P95 95.0, P99 99.0
+ | Metric1 P50 50.0, P90 90.0, P95 95.0, P99 99.0
|
""".trimMargin(),
summaryV1
@@ -131,7 +131,7 @@
assertEquals(
"""
|foo
- | Metric1 P50 50.0, P90 90.0, P95 95.0, P99 99.0
+ | Metric1 P50 50.0, P90 90.0, P95 95.0, P99 99.0
| Traces: Iteration [0](file://iter0.trace) [1](file://iter1.trace) [2](file://iter2.trace)
|
""".trimMargin(),
@@ -157,7 +157,7 @@
"""
|foo
| Metric1 min 0.0, median 1.0, max 2.0
- | Metric2 P50 50.0, P90 90.0, P95 95.0, P99 99.0
+ | Metric2 P50 50.0, P90 90.0, P95 95.0, P99 99.0
|
""".trimMargin(),
summaryV1
@@ -166,7 +166,7 @@
"""
|foo
| Metric1 [min 0.0](file://iter0.trace), [median 1.0](file://iter1.trace), [max 2.0](file://iter2.trace)
- | Metric2 P50 50.0, P90 90.0, P95 95.0, P99 99.0
+ | Metric2 P50 50.0, P90 90.0, P95 95.0, P99 99.0
| Traces: Iteration [0](file://iter0.trace) [1](file://iter1.trace) [2](file://iter2.trace)
|
""".trimMargin(),
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/FrameTimingQueryTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/FrameTimingQueryTest.kt
index df2b10f..4f6af3c 100644
--- a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/FrameTimingQueryTest.kt
+++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/FrameTimingQueryTest.kt
@@ -17,10 +17,9 @@
package androidx.benchmark.macro.perfetto
import androidx.benchmark.macro.createTempFileFromAsset
-import androidx.benchmark.macro.perfetto.FrameTimingQuery.FrameSubMetric.BasicFrameTime
-import androidx.benchmark.macro.perfetto.FrameTimingQuery.FrameSubMetric.FrameSlackTime
-import androidx.benchmark.macro.perfetto.FrameTimingQuery.FrameSubMetric.FrameTime
-import androidx.benchmark.macro.perfetto.FrameTimingQuery.FrameSubMetric.UiFrameTime
+import androidx.benchmark.macro.perfetto.FrameTimingQuery.SubMetric.FrameCpuTime
+import androidx.benchmark.macro.perfetto.FrameTimingQuery.SubMetric.FrameNegativeSlackTime
+import androidx.benchmark.macro.perfetto.FrameTimingQuery.SubMetric.FrameUiTime
import androidx.benchmark.perfetto.PerfettoHelper.Companion.isAbiSupported
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.MediumTest
@@ -45,8 +44,8 @@
assertEquals(
expected = mapOf(
- BasicFrameTime to listOf(9907605L, 6038595L, 4812136L, 3938490L),
- UiFrameTime to listOf(3086979L, 2868490L, 2232709L, 1889479L)
+ FrameCpuTime to listOf(9907605L, 6038595L, 4812136L, 3938490L),
+ FrameUiTime to listOf(3086979L, 2868490L, 2232709L, 1889479L)
),
actual = frameSubMetrics.mapValues {
it.value.subList(0, 4)
@@ -72,17 +71,16 @@
)
assertEquals(
expected = mapOf(
- FrameTime to listOf(15292863L, 8800138L, 6474705L, 8199845L),
- BasicFrameTime to listOf(6881407L, 5648542L, 3830261L, 4343438L),
- UiFrameTime to listOf(2965052L, 3246407L, 1562188L, 1945469L),
- FrameSlackTime to listOf(5207137L, 11699862L, 14025295L, 12300155L)
+ FrameCpuTime to listOf(6881407L, 5648542L, 3830261L, 4343438L),
+ FrameUiTime to listOf(2965052L, 3246407L, 1562188L, 1945469L),
+ FrameNegativeSlackTime to listOf(-5207137L, -11699862L, -14025295L, -12300155L)
),
actual = frameSubMetrics.mapValues {
it.value.subList(0, 4)
}
)
assertEquals(
- expected = List(4) { 96 },
+ expected = List(3) { 96 },
actual = frameSubMetrics.map { it.value.size },
message = "Expect same number of frames for each metric"
)
diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/IdeSummaryString.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/IdeSummaryString.kt
index 9868e6e..fd1de64 100644
--- a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/IdeSummaryString.kt
+++ b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/IdeSummaryString.kt
@@ -69,10 +69,10 @@
} +
measurements.sampledMetrics.map {
val name = it.name.padStart(maxLabelLength)
- val p50 = it.p50.toDisplayString()
- val p90 = it.p90.toDisplayString()
- val p95 = it.p95.toDisplayString()
- val p99 = it.p99.toDisplayString()
+ val p50 = it.p50.toDisplayString().padStart(maxValueLength)
+ val p90 = it.p90.toDisplayString().padStart(maxValueLength)
+ val p95 = it.p95.toDisplayString().padStart(maxValueLength)
+ val p99 = it.p99.toDisplayString().padStart(maxValueLength)
// we don't try and link percentiles, since they're grouped across multiple iters
" $name P50 $p50, P90 $p90, P95 $p95, P99 $p99"
}
diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/Metric.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/Metric.kt
index a6f127b..272b007 100644
--- a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/Metric.kt
+++ b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/Metric.kt
@@ -21,6 +21,7 @@
import androidx.annotation.RestrictTo
import androidx.benchmark.Shell
import androidx.benchmark.macro.perfetto.FrameTimingQuery
+import androidx.benchmark.macro.perfetto.FrameTimingQuery.SubMetric
import androidx.benchmark.macro.perfetto.PerfettoResultsParser.parseStartupResult
import androidx.benchmark.macro.perfetto.PerfettoTraceProcessor
import androidx.test.platform.app.InstrumentationRegistry
@@ -43,7 +44,13 @@
internal abstract fun getMetrics(packageName: String, tracePath: String): IterationResult
}
-public class FrameTimingMetric : Metric() {
+/**
+ * Legacy version of FrameTimingMetric, based on 'dumpsys gfxinfo' instead of trace data.
+ *
+ * Temporary - to be removed after transition to FrameTimingMetric
+ */
+@RestrictTo(RestrictTo.Scope.LIBRARY_GROUP_PREFIX)
+public class FrameTimingGfxInfoMetric : Metric() {
private lateinit var packageName: String
private val helper = JankCollectionHelper()
@@ -138,26 +145,33 @@
}
/**
- * WIP trace-based replacement for [FrameTimingMetric].
+ * Metric which captures timing information from frames produced by a benchmark, such as
+ * a scrolling or animation benchmark.
*/
-@RestrictTo(RestrictTo.Scope.LIBRARY_GROUP_PREFIX)
@Suppress("CanSealedSubClassBeObject")
-public class FrameTimingTraceMetric : Metric() {
+public class FrameTimingMetric : Metric() {
internal override fun configure(packageName: String) {}
internal override fun start() {}
internal override fun stop() {}
internal override fun getMetrics(packageName: String, tracePath: String): IterationResult {
- val frameTimesMs = FrameTimingQuery.getFrameSubMetrics(
+ val subMetricsMsMap = FrameTimingQuery.getFrameSubMetrics(
absoluteTracePath = tracePath,
captureApiLevel = Build.VERSION.SDK_INT,
packageName = packageName
- )[FrameTimingQuery.FrameSubMetric.FrameTime]!!
- .map { it / 1_000_000.0 } // Convert to ms
-
+ )
+ .filterKeys { it == SubMetric.FrameCpuTime || it == SubMetric.FrameNegativeSlackTime }
+ .mapKeys {
+ if (it.key == SubMetric.FrameCpuTime) "frameCpuTimeMs" else "frameNegativeSlackMs"
+ }
+ .mapValues { entry ->
+ entry.value.map { timeNs ->
+ timeNs / 1_000_000.0 // Convert to ms
+ }
+ }
return IterationResult(
singleMetrics = emptyMap(),
- sampledMetrics = mapOf("frameTimeMs" to frameTimesMs),
+ sampledMetrics = subMetricsMsMap,
timelineRangeNs = null
)
}
diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/FrameTimingQuery.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/FrameTimingQuery.kt
index 111329a..f4c5bf0 100644
--- a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/FrameTimingQuery.kt
+++ b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/FrameTimingQuery.kt
@@ -54,14 +54,13 @@
ORDER BY ts ASC
""".trimIndent()
- enum class FrameSubMetric {
- FrameTime,
- BasicFrameTime,
- UiFrameTime,
- FrameSlackTime;
+ enum class SubMetric {
+ FrameCpuTime,
+ FrameUiTime,
+ FrameNegativeSlackTime;
fun supportedOnApiLevel(apiLevel: Int): Boolean {
- return apiLevel >= 31 || this != FrameTime && this != FrameSlackTime
+ return apiLevel >= 31 || this != FrameNegativeSlackTime
}
}
@@ -83,12 +82,11 @@
val expectedSlice: Slice?,
val actualSlice: Slice?
) {
- fun get(subMetric: FrameSubMetric): Long {
+ fun get(subMetric: SubMetric): Long {
return when (subMetric) {
- FrameSubMetric.FrameTime -> actualSlice!!.dur
- FrameSubMetric.BasicFrameTime -> rtSlice.endTs - uiSlice.ts
- FrameSubMetric.UiFrameTime -> uiSlice.dur
- FrameSubMetric.FrameSlackTime -> expectedSlice!!.endTs - actualSlice!!.endTs
+ SubMetric.FrameCpuTime -> rtSlice.endTs - uiSlice.ts
+ SubMetric.FrameUiTime -> uiSlice.dur
+ SubMetric.FrameNegativeSlackTime -> actualSlice!!.endTs - expectedSlice!!.endTs
}
}
companion object {
@@ -140,7 +138,7 @@
absoluteTracePath: String,
captureApiLevel: Int,
packageName: String,
- ): Map<FrameSubMetric, List<Long>> {
+ ): Map<SubMetric, List<Long>> {
val queryResult = PerfettoTraceProcessor.rawQuery(
absoluteTracePath = absoluteTracePath,
query = getFullQuery(packageName)
@@ -199,7 +197,7 @@
}
}
- return FrameSubMetric.values()
+ return SubMetric.values()
.filter { it.supportedOnApiLevel(captureApiLevel) }
.associateWith { subMetric ->
frameData.map { frame -> frame.get(subMetric) }
diff --git a/benchmark/integration-tests/macrobenchmark/src/androidTest/java/androidx/benchmark/integration/macrobenchmark/TrivialListScrollBenchmark.kt b/benchmark/integration-tests/macrobenchmark/src/androidTest/java/androidx/benchmark/integration/macrobenchmark/TrivialListScrollBenchmark.kt
index 605f409..cb24edb 100644
--- a/benchmark/integration-tests/macrobenchmark/src/androidTest/java/androidx/benchmark/integration/macrobenchmark/TrivialListScrollBenchmark.kt
+++ b/benchmark/integration-tests/macrobenchmark/src/androidTest/java/androidx/benchmark/integration/macrobenchmark/TrivialListScrollBenchmark.kt
@@ -19,8 +19,8 @@
import android.content.Intent
import android.graphics.Point
import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.FrameTimingGfxInfoMetric
import androidx.benchmark.macro.FrameTimingMetric
-import androidx.benchmark.macro.FrameTimingTraceMetric
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.filters.LargeTest
import androidx.test.platform.app.InstrumentationRegistry
@@ -53,7 +53,7 @@
fun start() {
benchmarkRule.measureRepeated(
packageName = PACKAGE_NAME,
- metrics = listOf(FrameTimingMetric(), FrameTimingTraceMetric()),
+ metrics = listOf(FrameTimingMetric(), FrameTimingGfxInfoMetric()),
compilationMode = compilationMode,
iterations = 10,
setupBlock = {
diff --git a/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/NestedListsScrollBenchmark.kt b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/NestedListsScrollBenchmark.kt
index 0390523..4b3fe6b 100644
--- a/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/NestedListsScrollBenchmark.kt
+++ b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/NestedListsScrollBenchmark.kt
@@ -19,6 +19,7 @@
import android.content.Intent
import android.graphics.Point
import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.FrameTimingGfxInfoMetric
import androidx.benchmark.macro.FrameTimingMetric
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.filters.LargeTest
@@ -53,7 +54,7 @@
fun start() {
benchmarkRule.measureRepeated(
packageName = PACKAGE_NAME,
- metrics = listOf(FrameTimingMetric()),
+ metrics = listOf(FrameTimingMetric(), FrameTimingGfxInfoMetric()),
compilationMode = compilationMode,
iterations = 10,
setupBlock = {
diff --git a/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/TrivialListScrollBenchmark.kt b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/TrivialListScrollBenchmark.kt
index 08310ca..102d4ee 100644
--- a/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/TrivialListScrollBenchmark.kt
+++ b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/TrivialListScrollBenchmark.kt
@@ -19,6 +19,7 @@
import android.content.Intent
import android.graphics.Point
import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.FrameTimingGfxInfoMetric
import androidx.benchmark.macro.FrameTimingMetric
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.filters.LargeTest
@@ -53,7 +54,7 @@
fun start() {
benchmarkRule.measureRepeated(
packageName = PACKAGE_NAME,
- metrics = listOf(FrameTimingMetric()),
+ metrics = listOf(FrameTimingMetric(), FrameTimingGfxInfoMetric()),
compilationMode = compilationMode,
iterations = 10,
setupBlock = {
