Preparing to switch away from the deprecated test runner
A lot of ravenwood tests (and other tests too) use the deprecated
test runner androidx.test.runner.AndroidJUnit4, but it actually has
problems. We should switch them to
androidx.test.ext.junit.runners.AndroidJUnit4.
However, simply updating all tests could be problematic because:
- we need to update the build rules and add "androidx.test.ext.junit"
as a static dependency, which is probably safe but it could
cause class conflicts.(?)
- More importantly, the subtle difference between the test runner
could make the device side test fail.
So I'm going to do it in two steps:
- Step 1 -- this CL:
Add an optional validator to RavenwoodTest, which can be enabled
with RAVENWOOD_OPTIONAL_VALIDATION=1.
This makes a test fail if it's using the deprecated one.
- Added a test for this
- Added a script to replace the test runner.
Test: run-ravenwood-tests
Test: RAVENWOOD_OPTIONAL_VALIDATION=1 atest RavenwoodCoreTest
Bug: 317131861
Change-Id: I5762bd097c3a529ae6e14460d9e326ed5c6086b4
diff --git a/ravenwood/bulk_enable.py b/ravenwood/bulk_enable.py
index 83fda9e..aafaaff 100644
--- a/ravenwood/bulk_enable.py
+++ b/ravenwood/bulk_enable.py
@@ -21,7 +21,7 @@
classes that have partial success.
Typical usage:
-$ ENABLE_PROBE_IGNORED=1 atest MyTestsRavenwood
+$ RAVENWOOD_RUN_DISABLED_TESTS=1 atest MyTestsRavenwood
$ cd /path/to/tests/root
$ python bulk_enable.py /path/to/atest/output/host_log.txt
"""
diff --git a/ravenwood/coretest/Android.bp b/ravenwood/coretest/Android.bp
new file mode 100644
index 0000000..9b7f8f7
--- /dev/null
+++ b/ravenwood/coretest/Android.bp
@@ -0,0 +1,23 @@
+package {
+ // See: http://go/android-license-faq
+ // A large-scale-change added 'default_applicable_licenses' to import
+ // all of the 'license_kinds' from "frameworks_base_license"
+ // to get the below license kinds:
+ // SPDX-license-identifier-Apache-2.0
+ default_applicable_licenses: ["frameworks_base_license"],
+}
+
+android_ravenwood_test {
+ name: "RavenwoodCoreTest",
+
+ static_libs: [
+ "androidx.annotation_annotation",
+ "androidx.test.rules",
+ ],
+
+ srcs: [
+ "test/**/*.java",
+ ],
+ sdk_version: "test_current",
+ auto_gen_config: true,
+}
diff --git a/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java b/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java
new file mode 100644
index 0000000..e58c282
--- /dev/null
+++ b/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.coretest;
+
+import android.platform.test.ravenwood.RavenwoodRule;
+
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for the test runner validator in RavenwoodRule.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodTestRunnerValidationTest {
+ // Note the following rules don't have a @Rule, because they need to be applied in a specific
+ // order. So we use a RuleChain instead.
+ private ExpectedException mThrown = ExpectedException.none();
+ private final RavenwoodRule mRavenwood = new RavenwoodRule();
+
+ @Rule
+ public final RuleChain chain = RuleChain.outerRule(mThrown).around(mRavenwood);
+
+ public RavenwoodTestRunnerValidationTest() {
+ Assume.assumeTrue(mRavenwood._ravenwood_private$isOptionalValidationEnabled());
+ // Because RavenwoodRule will throw this error before executing the test method,
+ // we can't do it in the test method itself.
+ // So instead, we initialize it here.
+ mThrown.expectMessage("Switch to androidx.test.ext.junit.runners.AndroidJUnit4");
+ }
+
+ @Test
+ public void testValidateTestRunner() {
+ }
+}
diff --git a/ravenwood/fix_test_runner.py b/ravenwood/fix_test_runner.py
new file mode 100755
index 0000000..99b7a1f
--- /dev/null
+++ b/ravenwood/fix_test_runner.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tool switch the deprecated jetpack test runner to the correct one.
+
+Typical usage:
+$ RAVENWOOD_OPTIONAL_VALIDATION=1 atest MyTestsRavenwood # Prepend RAVENWOOD_RUN_DISABLED_TESTS=1 as needed
+$ cd /path/to/tests/root
+$ python bulk_enable.py /path/to/atest/output/host_log.txt
+"""
+
+import collections
+import os
+import re
+import subprocess
+import sys
+
+re_result = re.compile("I/ModuleListener.+?null-device-0 (.+?)#(.+?) ([A-Z_]+)(.*)$")
+
+OLD_RUNNER = "androidx.test.runner.AndroidJUnit4"
+NEW_RUNNER = "androidx.test.ext.junit.runners.AndroidJUnit4"
+SED_ARG = r"s/%s/%s/g" % (OLD_RUNNER, NEW_RUNNER)
+
+target = collections.defaultdict()
+
+with open(sys.argv[1]) as f:
+ for line in f.readlines():
+ result = re_result.search(line)
+ if result:
+ clazz, method, state, msg = result.groups()
+ if NEW_RUNNER in msg:
+ target[clazz] = 1
+
+if len(target) == 0:
+ print("No tests need updating.")
+ sys.exit(0)
+
+num_fixed = 0
+for clazz in target.keys():
+ print("Fixing test runner", clazz)
+ clazz_match = re.compile("%s\.(kt|java)" % (clazz.split(".")[-1]))
+ found = False
+ for root, dirs, files in os.walk("."):
+ for f in files:
+ if clazz_match.match(f):
+ found = True
+ num_fixed += 1
+ path = os.path.join(root, f)
+ subprocess.run(["sed", "-i", "-E", SED_ARG, path])
+ if not found:
+ print(f" Warining: tests {clazz} not found")
+
+
+print("Tests fixed", num_fixed)
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 8af561f..85fa65a 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -28,7 +28,10 @@
import com.android.internal.os.RuntimeInit;
+import org.junit.Assert;
import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.Statement;
import java.io.PrintStream;
import java.util.Map;
@@ -167,4 +170,31 @@
}
}
}
+
+ public static void validate(Statement base, Description description,
+ boolean enableOptionalValidation) {
+ validateTestRunner(base, description, enableOptionalValidation);
+ }
+
+ private static void validateTestRunner(Statement base, Description description,
+ boolean shouldFail) {
+ final var testClass = description.getTestClass();
+ final var runWith = testClass.getAnnotation(RunWith.class);
+ if (runWith == null) {
+ return;
+ }
+
+ // Due to build dependencies, we can't directly refer to androidx classes here,
+ // so just check the class name instead.
+ if (runWith.value().getCanonicalName().equals("androidx.test.runner.AndroidJUnit4")) {
+ var message = "Test " + testClass.getCanonicalName() + " uses deprecated"
+ + " test runner androidx.test.runner.AndroidJUnit4."
+ + " Switch to androidx.test.ext.junit.runners.AndroidJUnit4.";
+ if (shouldFail) {
+ Assert.fail(message);
+ } else {
+ System.err.println("Warning: " + message);
+ }
+ }
+ }
}
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
index daed457..764573d 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
@@ -89,6 +89,12 @@
private static final boolean ENABLE_REALLY_DISABLE_PATTERN =
!REALLY_DISABLE_PATTERN.pattern().isEmpty();
+ /**
+ * If true, enable optional validation on running tests.
+ */
+ private static final boolean ENABLE_OPTIONAL_VALIDATION = "1".equals(
+ System.getenv("RAVENWOOD_OPTIONAL_VALIDATION"));
+
static {
if (ENABLE_PROBE_IGNORED) {
System.out.println("$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
@@ -275,6 +281,12 @@
}
}
+ private void commonPrologue(Statement base, Description description) {
+ RavenwoodRuleImpl.logTestRunner("started", description);
+ RavenwoodRuleImpl.validate(base, description, ENABLE_OPTIONAL_VALIDATION);
+ RavenwoodRuleImpl.init(RavenwoodRule.this);
+ }
+
/**
* Run the given {@link Statement} with no special treatment.
*/
@@ -284,8 +296,7 @@
public void evaluate() throws Throwable {
Assume.assumeTrue(shouldEnableOnRavenwood(description));
- RavenwoodRuleImpl.logTestRunner("started", description);
- RavenwoodRuleImpl.init(RavenwoodRule.this);
+ commonPrologue(base, description);
try {
base.evaluate();
RavenwoodRuleImpl.logTestRunner("finished", description);
@@ -310,8 +321,7 @@
public void evaluate() throws Throwable {
Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
- RavenwoodRuleImpl.logTestRunner("started", description);
- RavenwoodRuleImpl.init(RavenwoodRule.this);
+ commonPrologue(base, description);
try {
base.evaluate();
} catch (Throwable t) {
@@ -331,4 +341,11 @@
}
};
}
+
+ /**
+ * Do not use it outside ravenwood core classes.
+ */
+ public boolean _ravenwood_private$isOptionalValidationEnabled() {
+ return ENABLE_OPTIONAL_VALIDATION;
+ }
}
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 7d172f2..e951351b 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -17,6 +17,7 @@
package android.platform.test.ravenwood;
import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
public class RavenwoodRuleImpl {
public static boolean isOnRavenwood() {
@@ -34,4 +35,8 @@
public static void logTestRunner(String label, Description description) {
// No-op when running on a real device
}
+
+ public static void validate(Statement base, Description description,
+ boolean enableOptionalValidation) {
+ }
}
diff --git a/ravenwood/minimum-test/Android.bp b/ravenwood/minimum-test/Android.bp
index bf3583c..e4ed3d5 100644
--- a/ravenwood/minimum-test/Android.bp
+++ b/ravenwood/minimum-test/Android.bp
@@ -13,6 +13,7 @@
static_libs: [
"androidx.annotation_annotation",
+ "androidx.test.ext.junit",
"androidx.test.rules",
],
diff --git a/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java b/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
index 7abfecf..03cfad5 100644
--- a/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
+++ b/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
@@ -18,7 +18,7 @@
import android.platform.test.annotations.IgnoreUnderRavenwood;
import android.platform.test.ravenwood.RavenwoodRule;
-import androidx.test.runner.AndroidJUnit4;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Assert;
import org.junit.Rule;
