Automatically greylist code in 3P packages
generate_hidden_api_lists now receives a file containing package names
which need to be greylisted (although it could be made to work with any
api list required).
Also took the opportunity to clean up the tests to reflect the more
strict code.
Bug: 129387816
Test: m appcompat
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I619f8581d166aa48eda572bc0053d8739d6420eb
diff --git a/config/hiddenapi-greylist-packages.txt b/config/hiddenapi-greylist-packages.txt
new file mode 100644
index 0000000..cae3bd9
--- /dev/null
+++ b/config/hiddenapi-greylist-packages.txt
@@ -0,0 +1,2 @@
+org.ccil.cowan.tagsoup
+org.ccil.cowan.tagsoup.jaxp
diff --git a/config/hiddenapi-greylist.txt b/config/hiddenapi-greylist.txt
index e3c7a2f..f43dcd2 100644
--- a/config/hiddenapi-greylist.txt
+++ b/config/hiddenapi-greylist.txt
@@ -2873,108 +2873,3 @@
Lorg/apache/xpath/XPathContext;->setCurrentNodeStack(Lorg/apache/xml/utils/IntStack;)V
Lorg/apache/xpath/XPathContext;->setSecureProcessing(Z)V
Lorg/apache/xpath/XPathContext;->setVarStack(Lorg/apache/xpath/VariableStack;)V
-Lorg/ccil/cowan/tagsoup/AttributesImpl;-><init>(Lorg/xml/sax/Attributes;)V
-Lorg/ccil/cowan/tagsoup/AttributesImpl;->addAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/AttributesImpl;->data:[Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/AttributesImpl;->length:I
-Lorg/ccil/cowan/tagsoup/AttributesImpl;->setAttribute(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/AttributesImpl;->setValue(ILjava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/AutoDetector;->autoDetectingReader(Ljava/io/InputStream;)Ljava/io/Reader;
-Lorg/ccil/cowan/tagsoup/Element;-><init>(Lorg/ccil/cowan/tagsoup/ElementType;Z)V
-Lorg/ccil/cowan/tagsoup/Element;->anonymize()V
-Lorg/ccil/cowan/tagsoup/Element;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
-Lorg/ccil/cowan/tagsoup/Element;->canContain(Lorg/ccil/cowan/tagsoup/Element;)Z
-Lorg/ccil/cowan/tagsoup/Element;->clean()V
-Lorg/ccil/cowan/tagsoup/Element;->flags()I
-Lorg/ccil/cowan/tagsoup/Element;->localName()Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Element;->name()Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Element;->namespace()Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Element;->next()Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Element;->parent()Lorg/ccil/cowan/tagsoup/ElementType;
-Lorg/ccil/cowan/tagsoup/Element;->preclosed:Z
-Lorg/ccil/cowan/tagsoup/Element;->setAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/Element;->setNext(Lorg/ccil/cowan/tagsoup/Element;)V
-Lorg/ccil/cowan/tagsoup/Element;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
-Lorg/ccil/cowan/tagsoup/Element;->theNext:Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Element;->theType:Lorg/ccil/cowan/tagsoup/ElementType;
-Lorg/ccil/cowan/tagsoup/ElementType;-><init>(Ljava/lang/String;IIILorg/ccil/cowan/tagsoup/Schema;)V
-Lorg/ccil/cowan/tagsoup/ElementType;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
-Lorg/ccil/cowan/tagsoup/ElementType;->setAttribute(Lorg/ccil/cowan/tagsoup/AttributesImpl;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/ElementType;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
-Lorg/ccil/cowan/tagsoup/ElementType;->theFlags:I
-Lorg/ccil/cowan/tagsoup/ElementType;->theLocalName:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/ElementType;->theMemberOf:I
-Lorg/ccil/cowan/tagsoup/ElementType;->theModel:I
-Lorg/ccil/cowan/tagsoup/ElementType;->theName:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/ElementType;->theNamespace:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/ElementType;->theParent:Lorg/ccil/cowan/tagsoup/ElementType;
-Lorg/ccil/cowan/tagsoup/ElementType;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
-Lorg/ccil/cowan/tagsoup/HTMLScanner;-><init>()V
-Lorg/ccil/cowan/tagsoup/HTMLSchema;-><init>()V
-Lorg/ccil/cowan/tagsoup/jaxp/SAXFactoryImpl;-><init>()V
-Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;-><init>()V
-Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;->newInstance(Ljava/util/Map;)Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;
-Lorg/ccil/cowan/tagsoup/Parser;-><init>()V
-Lorg/ccil/cowan/tagsoup/Parser;->bogonsEmpty:Z
-Lorg/ccil/cowan/tagsoup/Parser;->CDATAElements:Z
-Lorg/ccil/cowan/tagsoup/Parser;->cleanPublicid(Ljava/lang/String;)Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->defaultAttributes:Z
-Lorg/ccil/cowan/tagsoup/Parser;->etagchars:[C
-Lorg/ccil/cowan/tagsoup/Parser;->expandEntities(Ljava/lang/String;)Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->getInputStream(Ljava/lang/String;Ljava/lang/String;)Ljava/io/InputStream;
-Lorg/ccil/cowan/tagsoup/Parser;->ignorableWhitespace:Z
-Lorg/ccil/cowan/tagsoup/Parser;->ignoreBogons:Z
-Lorg/ccil/cowan/tagsoup/Parser;->lookupEntity([CII)I
-Lorg/ccil/cowan/tagsoup/Parser;->makeName([CII)Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->pop()V
-Lorg/ccil/cowan/tagsoup/Parser;->push(Lorg/ccil/cowan/tagsoup/Element;)V
-Lorg/ccil/cowan/tagsoup/Parser;->rectify(Lorg/ccil/cowan/tagsoup/Element;)V
-Lorg/ccil/cowan/tagsoup/Parser;->restart(Lorg/ccil/cowan/tagsoup/Element;)V
-Lorg/ccil/cowan/tagsoup/Parser;->restartablyPop()V
-Lorg/ccil/cowan/tagsoup/Parser;->rootBogons:Z
-Lorg/ccil/cowan/tagsoup/Parser;->schemaProperty:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->split(Ljava/lang/String;)[Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->theAttributeName:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->theAutoDetector:Lorg/ccil/cowan/tagsoup/AutoDetector;
-Lorg/ccil/cowan/tagsoup/Parser;->theContentHandler:Lorg/xml/sax/ContentHandler;
-Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeIsPresent:Z
-Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeSystemId:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->theFeatures:Ljava/util/HashMap;
-Lorg/ccil/cowan/tagsoup/Parser;->theLexicalHandler:Lorg/xml/sax/ext/LexicalHandler;
-Lorg/ccil/cowan/tagsoup/Parser;->theNewElement:Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Parser;->thePCDATA:Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Parser;->thePITarget:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->theSaved:Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Parser;->theScanner:Lorg/ccil/cowan/tagsoup/Scanner;
-Lorg/ccil/cowan/tagsoup/Parser;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
-Lorg/ccil/cowan/tagsoup/Parser;->theStack:Lorg/ccil/cowan/tagsoup/Element;
-Lorg/ccil/cowan/tagsoup/Parser;->trimquotes(Ljava/lang/String;)Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Parser;->virginStack:Z
-Lorg/ccil/cowan/tagsoup/PYXScanner;-><init>()V
-Lorg/ccil/cowan/tagsoup/PYXWriter;-><init>(Ljava/io/Writer;)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->aname([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->aval([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->entity([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->eof([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->etag([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->gi([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->pcdata([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->pi([CII)V
-Lorg/ccil/cowan/tagsoup/ScanHandler;->stagc([CII)V
-Lorg/ccil/cowan/tagsoup/Scanner;->startCDATA()V
-Lorg/ccil/cowan/tagsoup/Schema;->elementType(Ljava/lang/String;III)V
-Lorg/ccil/cowan/tagsoup/Schema;->getElementType(Ljava/lang/String;)Lorg/ccil/cowan/tagsoup/ElementType;
-Lorg/ccil/cowan/tagsoup/Schema;->getEntity(Ljava/lang/String;)I
-Lorg/ccil/cowan/tagsoup/Schema;->getPrefix()Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Schema;->getURI()Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Schema;->parent(Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/Schema;->theElementTypes:Ljava/util/HashMap;
-Lorg/ccil/cowan/tagsoup/Schema;->theEntities:Ljava/util/HashMap;
-Lorg/ccil/cowan/tagsoup/Schema;->thePrefix:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/Schema;->theRoot:Lorg/ccil/cowan/tagsoup/ElementType;
-Lorg/ccil/cowan/tagsoup/Schema;->theURI:Ljava/lang/String;
-Lorg/ccil/cowan/tagsoup/XMLWriter;-><init>(Ljava/io/Writer;)V
-Lorg/ccil/cowan/tagsoup/XMLWriter;->htmlMode:Z
-Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutput(Ljava/io/Writer;)V
-Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutputProperty(Ljava/lang/String;Ljava/lang/String;)V
-Lorg/ccil/cowan/tagsoup/XMLWriter;->setPrefix(Ljava/lang/String;Ljava/lang/String;)V
diff --git a/tools/hiddenapi/generate_hiddenapi_lists.py b/tools/hiddenapi/generate_hiddenapi_lists.py
index 6781eba..c856cc3 100755
--- a/tools/hiddenapi/generate_hiddenapi_lists.py
+++ b/tools/hiddenapi/generate_hiddenapi_lists.py
@@ -21,6 +21,7 @@
import os
import sys
import re
+import functools
# Names of flags recognized by the `hiddenapi` tool.
FLAG_WHITELIST = "whitelist"
@@ -58,6 +59,10 @@
# script to skip any entries which do not exist any more.
FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"
+# Suffix used in command line args to express that all apis within a given set
+# of packages should be assign the given flag.
+FLAG_PACKAGES_SUFFIX = "-packages"
+
# Regex patterns of fields/methods used in serialization. These are
# considered public API despite being hidden.
SERIALIZATION_PATTERNS = [
@@ -91,12 +96,16 @@
for flag in ALL_FLAGS:
ignore_conflicts_flag = flag + FLAG_IGNORE_CONFLICTS_SUFFIX
+ packages_flag = flag + FLAG_PACKAGES_SUFFIX
parser.add_argument('--' + flag, dest=flag, nargs='*', default=[], metavar='TXT_FILE',
help='lists of entries with flag "' + flag + '"')
parser.add_argument('--' + ignore_conflicts_flag, dest=ignore_conflicts_flag, nargs='*',
default=[], metavar='TXT_FILE',
help='lists of entries with flag "' + flag +
'". skip entry if missing or flag conflict.')
+ parser.add_argument('--' + packages_flag, dest=packages_flag, nargs='*',
+ default=[], metavar='TXT_FILE',
+ help='lists of packages to be added to ' + flag + ' list')
return parser.parse_args()
@@ -128,6 +137,19 @@
with open(filename, 'w') as f:
f.writelines(lines)
+def extract_package(signature):
+ """Extracts the package from a signature.
+
+ Args:
+ signature (string): JNI signature of a method or field.
+
+ Returns:
+ The package name of the class containing the field/method.
+ """
+ full_class_name = signature.split(";->")[0]
+ package_name = full_class_name[1:full_class_name.rindex("/")]
+ return package_name.replace('/', '.')
+
class FlagsDict:
def __init__(self):
self._dict_keyset = set()
@@ -206,7 +228,10 @@
self._dict_keyset.update([ csv[0] for csv in csv_values ])
# Check that all flags are known.
- csv_flags = set(reduce(lambda x, y: set(x).union(y), [ csv[1:] for csv in csv_values ], []))
+ csv_flags = set(functools.reduce(
+ lambda x, y: set(x).union(y),
+ [ csv[1:] for csv in csv_values ],
+ []))
self._check_flags_set(csv_flags, source)
# Iterate over all CSV lines, find entry in dict and append flags to it.
@@ -273,6 +298,15 @@
valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(filename))
flags.assign_flag(flag, valid_entries, filename)
+ # All members in the specified packages will be assigned the appropriate flag.
+ for flag in ALL_FLAGS:
+ for filename in args[flag + FLAG_PACKAGES_SUFFIX]:
+ packages_needing_list = set(read_lines(filename))
+ should_add_signature_to_list = lambda sig,lists: extract_package(
+ sig) in packages_needing_list and not lists
+ valid_entries = flags.filter_apis(should_add_signature_to_list)
+ flags.assign_flag(flag, valid_entries)
+
# Assign all remaining entries to the blacklist.
flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))
diff --git a/tools/hiddenapi/generate_hiddenapi_lists_test.py b/tools/hiddenapi/generate_hiddenapi_lists_test.py
index 249f37d..4dc880b 100755
--- a/tools/hiddenapi/generate_hiddenapi_lists_test.py
+++ b/tools/hiddenapi/generate_hiddenapi_lists_test.py
@@ -18,33 +18,23 @@
from generate_hiddenapi_lists import *
class TestHiddenapiListGeneration(unittest.TestCase):
- def test_init(self):
- # Check empty lists
- flags = FlagsDict([], [])
- self.assertEquals(flags.generate_csv(), [])
-
- # Check valid input - two public and two private API signatures.
- flags = FlagsDict(['A', 'B'], ['C', 'D'])
- self.assertEquals(flags.generate_csv(),
- [ 'A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, 'C', 'D' ])
-
- # Check invalid input - overlapping public/private API signatures.
- with self.assertRaises(AssertionError):
- flags = FlagsDict(['A', 'B'], ['B', 'C', 'D'])
def test_filter_apis(self):
# Initialize flags so that A and B are put on the whitelist and
# C, D, E are left unassigned. Try filtering for the unassigned ones.
- flags = FlagsDict(['A', 'B'], ['C', 'D', 'E'])
+ flags = FlagsDict()
+ flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,
+ 'C', 'D', 'E'])
filter_set = flags.filter_apis(lambda api, flags: not flags)
self.assertTrue(isinstance(filter_set, set))
self.assertEqual(filter_set, set([ 'C', 'D', 'E' ]))
def test_get_valid_subset_of_unassigned_keys(self):
# Create flags where only A is unassigned.
- flags = FlagsDict(['A'], ['B', 'C'])
+ flags = FlagsDict()
+ flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])
flags.assign_flag(FLAG_GREYLIST, set(['C']))
- self.assertEquals(flags.generate_csv(),
+ self.assertEqual(flags.generate_csv(),
[ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])
# Check three things:
@@ -55,44 +45,30 @@
flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ]))
def test_parse_and_merge_csv(self):
- flags = FlagsDict(['A'], ['B'])
- self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+ flags = FlagsDict()
# Test empty CSV entry.
- flags.parse_and_merge_csv(['B'])
- self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
-
- # Test assigning an already assigned flag.
- flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST])
- self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+ self.assertEqual(flags.generate_csv(), [])
# Test new additions.
flags.parse_and_merge_csv([
'A,' + FLAG_GREYLIST,
'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O ])
self.assertEqual(flags.generate_csv(),
- [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST,
+ [ 'A,' + FLAG_GREYLIST,
'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O ])
- # Test unknown API signature.
- with self.assertRaises(AssertionError):
- flags.parse_and_merge_csv([ 'C' ])
-
# Test unknown flag.
with self.assertRaises(AssertionError):
- flags.parse_and_merge_csv([ 'A,foo' ])
+ flags.parse_and_merge_csv([ 'C,foo' ])
def test_assign_flag(self):
- flags = FlagsDict(['A'], ['B'])
- self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
-
- # Test assigning an already assigned flag.
- flags.assign_flag(FLAG_WHITELIST, set([ 'A' ]))
- self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
+ flags = FlagsDict()
+ flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])
# Test new additions.
flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))
- self.assertEquals(flags.generate_csv(),
+ self.assertEqual(flags.generate_csv(),
[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])
# Test invalid API signature.
@@ -103,5 +79,18 @@
with self.assertRaises(AssertionError):
flags.assign_flag('foo', set([ 'A' ]))
+ def test_extract_package(self):
+ signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;'
+ expected_package = 'com.foo.bar'
+ self.assertEqual(extract_package(signature), expected_package)
+
+ signature = 'Lcom/foo1/bar/MyClass;->method2()V'
+ expected_package = 'com.foo1.bar'
+ self.assertEqual(extract_package(signature), expected_package)
+
+ signature = 'Lcom/foo_bar/baz/MyClass;->method3()V'
+ expected_package = 'com.foo_bar.baz'
+ self.assertEqual(extract_package(signature), expected_package)
+
if __name__ == '__main__':
unittest.main()
