Reset Compilation cleanly, without reinstall
Fixes: 249143766
Test: CompilationConsistencyBenchmark
Test: CompilationModeTest
Relnote: " No longer uninstall and reinstall target app to reset
compilation. With further investigation, the sole reason this was
necessary was to reset shader state, which has been fixed
separately. Now the target app remains installed, even without root,
without affecting app data. "
Change-Id: I8405b59a85e04f9ae3eeee0e3eecedcc47241e02
diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/CompilationMode.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/CompilationMode.kt
index 39cf318..285827b 100644
--- a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/CompilationMode.kt
+++ b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/CompilationMode.kt
@@ -28,16 +28,14 @@
import androidx.benchmark.macro.CompilationMode.Ignore
import androidx.benchmark.macro.CompilationMode.None
import androidx.benchmark.macro.CompilationMode.Partial
-import androidx.benchmark.userspaceTrace
import androidx.profileinstaller.ProfileInstallReceiver
import org.junit.AssumptionViolatedException
/**
* Type of compilation to use for a Macrobenchmark.
*
- * Every Macrobenchmark has compilation reset before running, so that previous runs do not interfere
- * with the next. This compilation mode dictates any pre-compilation that occurs before repeatedly
- * running the setup / measure blocks of the benchmark.
+ * This compilation mode controls pre-compilation that occurs before running the setup / measure
+ * blocks of the benchmark.
*
* On Android N+ (API 24+), there are different levels of compilation supported:
*
@@ -48,9 +46,12 @@
* guide pre-compilation to mimic an application's performance after some, and JIT-ing has occurred.
*
* * [Full] - the app is fully pre-compiled. This is generally not representative of real user
- * experience, as apps are not fully pre-compiled on user devices, but this can be used to either
- * illustrate ideal performance, or to reduce noise/inconsistency from just-in-time compilation
- * while the benchmark runs.
+ * experience, as apps are not fully pre-compiled on user devices more recent than Android N
+ * (API 24). `Full` can be used to show unrealistic but potentially more stable performance by
+ * removing the noise/inconsistency from just-in-time compilation within benchmark runs. Note that
+ * `Full` compilation will often be slower than [Partial] compilation, as the increased code size
+ * creates more cost for disk loading during startup, and increases pressure in the instruction
+ * cache.
*
* * [None] - the app isn't pre-compiled at all, bypassing the default compilation that should
* generally be done at install time, e.g. by the Play Store. This will illustrate worst case
@@ -58,7 +59,8 @@
* useful for judging the performance impact of the baseline profiles included in your application.
*
* * [Ignore] - the state of compilation will be ignored. The intended use-case is for a developer
- * to customize the compilation state for an app; and then tell Macrobenchmark to leave it unchanged.
+ * to customize the compilation state for an app; and then tell Macrobenchmark to leave it
+ * unchanged.
*
* On Android M (API 23), only [Full] is supported, as all apps are always fully compiled.
*
@@ -71,40 +73,12 @@
sealed class CompilationMode {
internal fun resetAndCompile(
packageName: String,
- usePackageReset: () -> Boolean = Shell::isSessionRooted,
killProcessBlock: () -> Unit,
warmupBlock: () -> Unit
) {
if (Build.VERSION.SDK_INT >= 24) {
if (Arguments.enableCompilation) {
Log.d(TAG, "Resetting $packageName")
- // The compilation mode chooses whether a reset is required or not.
- // Currently the only compilation mode that does not perform a reset is
- // CompilationMode.Ignore.
- if (shouldReset()) {
- // It's not possible to reset the compilation profile on `user` builds.
- // The flag `enablePackageReset` can be set to `true` on `userdebug` builds in
- // order to speed-up the profile reset. When set to false, reset is performed
- // uninstalling and reinstalling the app.
- if (usePackageReset()) {
- // Package reset enabled
- Log.d(TAG, "Re-compiling $packageName")
- // cmd package compile --reset returns a "Success" or a "Failure" to stdout.
- // Rather than rely on exit codes which are not always correct, we specifically
- // look for the work "Success" in stdout to make sure reset actually
- // happened.
- val output = Shell.executeScriptWithStderr(
- "cmd package compile --reset $packageName"
- )
- check(output.stdout.trim() == "Success") {
- "Unable to recompile $packageName ($output)"
- }
- } else {
- // User builds. Kick off a full uninstall-reinstall
- Log.d(TAG, "Reinstalling $packageName")
- reinstallPackage(packageName)
- }
- }
// Write skip file to stop profile installer from interfering with the benchmark
writeProfileInstallerSkipFile(packageName, killProcessBlock = killProcessBlock)
compileImpl(packageName, killProcessBlock, warmupBlock)
@@ -114,46 +88,6 @@
}
}
- // This is a more expensive when compared to `compile --reset`.
- private fun reinstallPackage(packageName: String) {
- userspaceTrace("reinstallPackage") {
- val packagePath = Shell.executeScript("pm path $packageName")
- // The result looks like: `package: <result>`
- val apkPath = packagePath.substringAfter("package:").trim()
- // Copy the APK to /data/local/temp
- val tempApkPath = "/data/local/tmp/$packageName-${System.currentTimeMillis()}.apk"
- Log.d(TAG, "Copying APK to $tempApkPath")
- val result = Shell.executeScriptWithStderr(
- "cp $apkPath $tempApkPath"
- )
- if (result.stderr.isNotBlank()) {
- Log.w(TAG, "Unable to copy apk ($result)")
- } else {
- try {
- // Uninstall package
- // This is what effectively clears the ART profiles
- Log.d(TAG, "Uninstalling $packageName")
- var output = Shell.executeScriptWithStderr("pm uninstall $packageName")
- check(output.stdout.trim() == "Success") {
- "Unable to uninstall $packageName ($result)"
- }
- // Install the APK from /data/local/tmp
- Log.d(TAG, "Installing $packageName")
- // Provide a `-t` argument to `pm install` to ensure test packages are
- // correctly installed. (b/231294733)
- output = Shell.executeScriptWithStderr("pm install -t $tempApkPath")
- check(output.stdout.trim() == "Success") {
- "Unable to install $packageName ($result)"
- }
- } finally {
- // Cleanup the temporary APK
- Log.d(TAG, "Deleting $tempApkPath")
- Shell.executeCommand("rm $tempApkPath")
- }
- }
- }
- }
-
/**
* Writes a skip file via a [ProfileInstallReceiver] broadcast, so profile installation
* does not interfere with benchmarks.
@@ -180,9 +114,6 @@
warmupBlock: () -> Unit
)
- @RequiresApi(24)
- internal abstract fun shouldReset(): Boolean
-
/**
* No pre-compilation - a compilation profile reset is performed and the entire app will be
* allowed to Just-In-Time compile as it runs.
@@ -200,10 +131,8 @@
killProcessBlock: () -> Unit,
warmupBlock: () -> Unit
) {
- // nothing to do!
+ cmdPackageCompile(packageName, "verify")
}
-
- override fun shouldReset(): Boolean = true
}
/**
@@ -223,8 +152,6 @@
) {
// Do nothing.
}
-
- override fun shouldReset(): Boolean = false
}
/**
@@ -335,8 +262,6 @@
cmdPackageCompile(packageName, "speed-profile")
}
}
-
- override fun shouldReset(): Boolean = true
}
/**
@@ -361,8 +286,6 @@
}
// Noop on older versions: apps are fully compiled at install time on API 23 and below
}
-
- override fun shouldReset(): Boolean = true
}
/**
@@ -386,8 +309,6 @@
) {
// Nothing to do - handled externally
}
-
- override fun shouldReset(): Boolean = true
}
companion object {
diff --git a/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/CompilationConsistencyBenchmark.kt b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/CompilationConsistencyBenchmark.kt
new file mode 100644
index 0000000..90fa1d7
--- /dev/null
+++ b/compose/integration-tests/macrobenchmark/src/androidTest/java/androidx/compose/integration/macrobenchmark/CompilationConsistencyBenchmark.kt
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.compose.integration.macrobenchmark
+
+import android.os.Build
+import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+import androidx.test.filters.LargeTest
+import androidx.test.filters.SdkSuppress
+import androidx.testutils.COMPILATION_MODES
+import androidx.testutils.measureStartup
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+/**
+ * This benchmark is used to verify that compilation mode performance is consistent across runs,
+ * and that compilation state doesn't leak across benchmarks.
+ */
+@LargeTest
+@SdkSuppress(minSdkVersion = Build.VERSION_CODES.N)
+@RunWith(Parameterized::class)
+class CompilationConsistencyBenchmark(
+ @Suppress("UNUSED_PARAMETER") iteration: Int,
+ private val compilationMode: CompilationMode
+) {
+ @get:Rule
+ val benchmarkRule = MacrobenchmarkRule()
+
+ @Test
+ fun startup() = benchmarkRule.measureStartup(
+ compilationMode = compilationMode,
+ startupMode = StartupMode.COLD,
+ packageName = packageName
+ ) {
+ action = "androidx.compose.integration.macrobenchmark.target.LAZY_COLUMN_ACTIVITY"
+ putExtra("ITEM_COUNT", 5)
+ }
+
+ companion object {
+ val packageName = "androidx.compose.integration.macrobenchmark.target"
+
+ @Parameterized.Parameters(name = "iter={0},compilation={1}")
+ @JvmStatic
+ fun parameters(): List<Array<Any>> = mutableListOf<Array<Any>>().apply {
+ for (iter in 1..4) {
+ for (compilationMode in COMPILATION_MODES) {
+ add(arrayOf(iter, compilationMode))
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
