blob: e39cf958f40d9a61c4b23ac224ae47fd223caa97 [file] [log] [blame]
Omar Ismail95b79e72024-09-11 09:47:52 -04001#!/usr/bin/env python3
Jeff Gastona7df0922021-06-02 17:39:47 -04002
3"""
4Parses information about failing tests, and then generates a change to disable them.
5
6Requires that the `bugged` command-line tool is installed, see go/bugged .
7"""
8
9import argparse, csv, os, subprocess
10
11parser = argparse.ArgumentParser(
12 description=__doc__
13)
Jeff Gastona7c77722023-11-02 15:15:59 -040014parser.add_argument("-v", help="Verbose", action="store_true")
Jeff Gastona7df0922021-06-02 17:39:47 -040015
16dirOfThisScript = os.path.dirname(os.path.realpath(__file__))
17supportRoot = os.path.dirname(dirOfThisScript)
18
Jeff Gastona7c77722023-11-02 15:15:59 -040019logger = None
20
21class PrintLogger(object):
22 def log(self, message):
23 print(message)
24
25class DisabledLogger(object):
26 def log(self, message):
27 pass
28
29def log(message):
30 logger.log(message)
31
Jeff Gastona7df0922021-06-02 17:39:47 -040032class LocatedFailure(object):
Jeff Gastona7c77722023-11-02 15:15:59 -040033 def __init__(self, failure, location, bugId):
Jeff Gastona7df0922021-06-02 17:39:47 -040034 self.failure = failure
35 self.location = location
Jeff Gastona7c77722023-11-02 15:15:59 -040036 self.bugId = bugId
Jeff Gastona7df0922021-06-02 17:39:47 -040037
38class TestFailure(object):
Jeff Gastona7c77722023-11-02 15:15:59 -040039 def __init__(self, qualifiedClassName, methodName, testDefinitionName, branchName, testFailureUrl, bugId):
Jeff Gastona7df0922021-06-02 17:39:47 -040040 self.qualifiedClassName = qualifiedClassName
41 self.methodName = methodName
42 self.testDefinitionName = testDefinitionName
Jeff Gastona7df0922021-06-02 17:39:47 -040043 self.branchName = branchName
Jeff Gastona7c77722023-11-02 15:15:59 -040044 self.failureUrl = testFailureUrl
45 self.bugId = bugId
Jeff Gastona7df0922021-06-02 17:39:47 -040046
47 def getUrl(self):
Jeff Gastona7c77722023-11-02 15:15:59 -040048 return self.testFailureUrl
Jeff Gastona7df0922021-06-02 17:39:47 -040049
50class FailuresDatabase(object):
51 """A collection of LocatedFailure instances, organized by their locations"""
52 def __init__(self):
53 self.failuresByPath = {}
54
55 def add(self, locatedFailure):
56 path = locatedFailure.location.filePath
57 if path not in self.failuresByPath:
58 self.failuresByPath[path] = {}
59 failuresAtPath = self.failuresByPath[path]
60
61 lineNumber = locatedFailure.location.lineNumber
62 if lineNumber not in failuresAtPath:
63 failuresAtPath[lineNumber] = locatedFailure
Jeff Gastona7df0922021-06-02 17:39:47 -040064
65 # returns Map<String, LocatedFailure> with key being filePath
66 def getAll(self):
67 results = {}
68 for path, failuresAtPath in self.failuresByPath.items():
69 lineNumbers = sorted(failuresAtPath.keys(), reverse=True)
70 resultsAtPath = []
71 # add failures in reverse order to make it easier to modify methods without adjusting line numbers for other methods
72 for line in lineNumbers:
73 resultsAtPath.append(failuresAtPath[line])
74 results[path] = resultsAtPath
75 return results
76
Jeff Gastona7c77722023-11-02 15:15:59 -040077def parseBugLine(bugId, line):
78 components = line.split(" | ")
79 if len(components) < 3:
80 return None
81 testLink = components[1]
82 # Example test link: [compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl](https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
83 closeBracketIndex = testLink.rindex("]")
84 if closeBracketIndex <= 0:
85 raise Exception("Failed to parse b/" + bugId + " '" + line + "', testLink '" + testLink + "', closeBracketIndex = " + str(closeBracketIndex))
86 linkText = testLink[1:closeBracketIndex]
87 linkDest = testLink[closeBracketIndex + 1:]
88 # Example linkText: compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
89 # Example linkDest: (https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
90 testResultUrl = linkDest.replace("(", "").replace(")", "")
91 # Example testResultUrl: https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098
92 spaceIndex = linkText.index(" ")
93 if spaceIndex <= 0:
94 raise Exception("Failed to parse b/" + bugId + " '" + line + "', linkText = '" + linkText + ", spaceIndex = " + str(spaceIndex))
95 testDefinitionName = linkText[:spaceIndex]
96 testPath = linkText[spaceIndex+1:]
97 # Example test path: androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
98 testPathSplit = testPath.split("#")
99 if len(testPathSplit) != 2:
100 raise Exception("Failed to parse b/" + bugId + " '" + line + "', testPath = '" + testPath + "', len(testPathSplit) = " + str(len(testPathSplit)))
101 testClass, testMethod = testPathSplit
102
103 branchName = components[2].strip()
104 print(" parsed test failure class=" + testClass + " method='" + testMethod + "' definition=" + testDefinitionName + " branch=" + branchName + " failureUrl=" + testResultUrl + " bugId=" + bugId)
105 return TestFailure(testClass, testMethod, testDefinitionName, branchName, testResultUrl, bugId)
106
107def parseBug(bugId):
108 bugText = shellRunner.runAndGetOutput(["bugged", "show", bugId])
109 log("bug text = '" + bugText + "'")
Jeff Gastona7df0922021-06-02 17:39:47 -0400110 failures = []
Jeff Gastona7c77722023-11-02 15:15:59 -0400111 bugLines = bugText.split("\n")
112
113 stillFailing = True
114 listingTests = False
115 for i in range(len(bugLines)):
116 line = bugLines[i]
117 #log("parsing bug line " + line)
118 if listingTests:
119 failure = parseBugLine(bugId, line)
120 if failure is not None:
121 failures.append(failure)
122 if "---|---|---|---|---" in line: # table start
123 listingTests = True
124 if " # " in line: # start of new section
125 listingTests = False
126 if "There are no more failing tests in this regression" in line or "ATN has not seen a failure for this regression recently." in line or "This regression has been identified as a duplicate of another one" in line:
127 stillFailing = False
128 if len(failures) < 1:
129 raise Exception("Failed to parse b/" + bugId + ": identified 0 failures. Rerun with -v for more information")
130 if not stillFailing:
131 print("tests no longer failing")
132 return []
133 return failures
134
135# identifies failing tests
136def getFailureData():
137 bugsQuery = ["bugged", "search", "hotlistid:5083126 status:open", "--columns", "issue"]
138 print("Searching for bugs: " + str(bugsQuery))
139 bugsOutput = shellRunner.runAndGetOutput(bugsQuery)
140 bugIds = bugsOutput.split("\n")
141 print("Checking " + str(len(bugIds)) + " bugs")
142 failures = []
143 for i in range(len(bugIds)):
144 bugId = bugIds[i].strip()
145 if bugId != "issue" and bugId != "":
146 print("")
147 print("Parsing bug " + bugId + " (" + str(i) + "/" + str(len(bugIds)) + ")")
148 failures += parseBug(bugId)
Jeff Gastona7df0922021-06-02 17:39:47 -0400149 return failures
150
151class FileLocation(object):
152 def __init__(self, filePath, lineNumber):
153 self.filePath = filePath
154 self.lineNumber = lineNumber
155
156 def __str__(self):
157 return self.filePath + "#" + str(self.lineNumber)
158
159class ShellRunner(object):
160 def __init__(self):
161 return
162
163 def runAndGetOutput(self, args):
164 result = subprocess.run(args, capture_output=True, text=True).stdout
165 return result
166
167 def run(self, args):
168 subprocess.run(args, capture_output=False)
169
170shellRunner = ShellRunner()
171
172class FileFinder(object):
173 def __init__(self, rootPath):
174 self.rootPath = rootPath
175 self.resultsCache = {}
176
177 def findIname(self, name):
178 if name not in self.resultsCache:
179 text = shellRunner.runAndGetOutput(["find", self.rootPath , "-type", "f", "-iname", name])
180 filePaths = [path.strip() for path in text.split("\n")]
181 filePaths = [path for path in filePaths if path != ""]
182 self.resultsCache[name] = filePaths
183 return self.resultsCache[name]
184fileFinder = FileFinder(supportRoot)
185
186class ClassFinder(object):
187 """Locates the file path and line number for classes and methods"""
188 def __init__(self):
189 self.classToFile_cache = {}
190 self.methodLocations_cache = {}
191
192 def findMethod(self, qualifiedClassName, methodName):
193 bracketIndex = methodName.find("[")
194 if bracketIndex >= 0:
195 methodName = methodName[:bracketIndex]
196 fullName = qualifiedClassName + "." + methodName
197 containingFile = self.findFileContainingClass(qualifiedClassName)
198 if containingFile is None:
199 return None
200 if fullName not in self.methodLocations_cache:
201 index = -1
202 foundLineNumber = None
203 with open(containingFile) as f:
204 for line in f:
205 index += 1
206 if (" " + methodName + "(") in line:
207 if foundLineNumber is not None:
208 # found two matches, can't choose one
209 foundLineNumber = None
210 break
211 foundLineNumber = index
212 result = None
213 if foundLineNumber is not None:
214 result = FileLocation(containingFile, foundLineNumber)
215 self.methodLocations_cache[fullName] = result
216 return self.methodLocations_cache[fullName]
217
218
219 def findFileContainingClass(self, qualifiedName):
220 if qualifiedName not in self.classToFile_cache:
221 lastDotIndex = qualifiedName.rindex(".")
222 if lastDotIndex >= 0:
223 packageName = qualifiedName[:lastDotIndex]
224 className = qualifiedName[lastDotIndex + 1:]
225 else:
226 packageName = ""
227 className = qualifiedName
228 options = fileFinder.findIname(className + ".*")
229 possibleContainingFiles = sorted(options)
230 result = None
231 for f in possibleContainingFiles:
232 if self.getPackage(f) == packageName:
233 result = f
234 break
235 self.classToFile_cache[qualifiedName] = result
236 return self.classToFile_cache[qualifiedName]
237
238 def getPackage(self, filePath):
239 prefix = "package "
240 with open(filePath) as f:
241 for line in f:
242 line = line.strip()
243 if line.startswith(prefix):
244 suffix = line[len(prefix):]
245 if suffix.endswith(";"):
246 return suffix[:-1]
247 return suffix
248 return None
249
250classFinder = ClassFinder()
251
252def readFile(path):
253 f = open(path)
254 text = f.read()
255 f.close()
256 return text
257
258def writeFile(path, text):
259 f = open(path, "w")
260 f.write(text)
261 f.close()
262
263def extractIndent(text):
264 indentSize = 0
265 for c in text:
266 if c == " ":
267 indentSize += 1
268 else:
269 break
270 return " " * indentSize
271
272class SourceFile(object):
273 """An in-memory model of a source file (java, kotlin) that can be manipulated and saved"""
274 def __init__(self, path):
275 text = readFile(path)
276 self.lines = text.split("\n")
277 self.path = path
278
279 def isKotlin(self):
280 return self.path.endswith(".kt")
281
282 def maybeSemicolon(self):
283 if self.isKotlin():
284 return ""
285 return ";"
286
287 def addAnnotation(self, methodLineNumber, annotation):
288 parenIndex = annotation.find("(")
289 if parenIndex > 0:
290 baseName = annotation[:parenIndex]
291 else:
292 baseName = annotation
293 if self.findAnnotationLine(methodLineNumber, baseName) is not None:
294 # already have an annotation, don't need to add another
295 return
296 indent = extractIndent(self.lines[methodLineNumber])
297 self.insertLine(methodLineNumber, indent + annotation)
298
299 # Adds an import to this file
300 # Attempts to preserve alphabetical import ordering:
301 # If two consecutive imports can be found such that one should precede this import and
302 # one should follow this import, inserts between those two imports
303 # Otherwise attempts to add this import after the last import or before the first import
304 # (Note that imports might be grouped into multiple blocks, each separated by a blank line)
305 def addImport(self, symbolText):
306 insertText = "import " + symbolText + self.maybeSemicolon()
307 if insertText in self.lines:
308 return # already added
309 # set of lines that the insertion could immediately precede
310 beforeLineNumbers = set()
311 # set of lines that the insertion could immediately follow
312 afterLineNumbers = set()
313 for i in range(len(self.lines)):
314 line = self.lines[i]
315 if line.startswith("import"):
316 # found an import. Should our import be before or after?
317 if insertText < line:
318 beforeLineNumbers.add(i)
319 else:
320 afterLineNumbers.add(i)
321 # search for two adjacent lines that the line can be inserted between
322 insertionLineNumber = None
323 for i in range(len(self.lines) - 1):
324 if i in afterLineNumbers and (i + 1) in beforeLineNumbers:
325 insertionLineNumber = i + 1
326 break
327 # search for a line we can insert after
328 if insertionLineNumber is None:
329 for i in range(len(self.lines) - 1):
330 if i in afterLineNumbers and (i + 1) not in afterLineNumbers:
331 insertionLineNumber = i + 1
332 break
333 # search for a line we can insert before
334 if insertionLineNumber is None:
335 for i in range(len(self.lines) - 1, 0, -1):
336 if i in beforeLineNumbers and (i - 1) not in beforeLineNumbers:
337 insertionLineNumber = i
338 break
339
340 if insertionLineNumber is not None:
341 self.insertLine(insertionLineNumber, insertText)
342
343 def insertLine(self, beforeLineNumber, text):
344 self.lines = self.lines[:beforeLineNumber] + [text] + self.lines[beforeLineNumber:]
345
346 def findAnnotationLine(self, methodLineNumber, annotationText):
347 lineNumber = methodLineNumber
348 while True:
349 if lineNumber < 0:
350 return None
351 if annotationText in self.lines[lineNumber]:
352 return lineNumber
353 if self.lines[lineNumber].strip() == "":
354 return None
355 lineNumber -= 1
356
357 def removeLine(self, index):
358 self.lines = self.lines[:index] + self.lines[index + 1:]
359
360 def hasAnnotation(self, methodLineNumber, annotation):
361 return self.findAnnotationLine(methodLineNumber, annotation) is not None
362
363 def save(self):
364 writeFile(self.path, "\n".join(self.lines))
365
Jeff Gastona7df0922021-06-02 17:39:47 -0400366# converts from a List<TestFailure> to a FailuresDatabase containing LocatedFailure
367def locate(failures):
368 db = FailuresDatabase()
369 for failure in failures:
370 location = classFinder.findMethod(failure.qualifiedClassName, failure.methodName)
371 if location is not None:
Jeff Gastona7c77722023-11-02 15:15:59 -0400372 db.add(LocatedFailure(failure, location, failure.bugId))
Jeff Gastona7df0922021-06-02 17:39:47 -0400373 else:
Jeff Gastona7c77722023-11-02 15:15:59 -0400374 message = "Could not locate " + str(failure.qualifiedClassName) + "#" + str(failure.methodName) + " for " + str(failure.bugId)
Jeff Gastona7df0922021-06-02 17:39:47 -0400375 if failure.branchName != "aosp-androidx-main":
376 message += ", should be in " + failure.branchName
377 print(message)
378 return db
379
Jeff Gastona7df0922021-06-02 17:39:47 -0400380# Given a FailureDatabase, disables all of the tests mentioned in it, by adding the appropriate
381# annotations:
Jeff Gastona7c77722023-11-02 15:15:59 -0400382# failures get annotated with @Ignore ,
Jeff Gastona7df0922021-06-02 17:39:47 -0400383# Annotations link to the associated bug if possible
384def disable(failuresDatabase):
Jeff Gastona7df0922021-06-02 17:39:47 -0400385 numUpdates = 0
386 failuresByPath = failuresDatabase.getAll()
387 for path, failuresAtPath in failuresByPath.items():
388 source = SourceFile(path)
389 addedIgnore = False
Jeff Gastona7df0922021-06-02 17:39:47 -0400390 for failure in failuresAtPath:
391 lineNumber = failure.location.lineNumber
Jeff Gastona7c77722023-11-02 15:15:59 -0400392 if source.hasAnnotation(lineNumber, "@Ignore"):
Jeff Gastona7df0922021-06-02 17:39:47 -0400393 continue
Jeff Gastona7c77722023-11-02 15:15:59 -0400394 bugId = failure.bugId
395 bugText = '"b/' + bugId + '"'
396 source.addAnnotation(lineNumber, "@Ignore(" + bugText + ")")
397 addedIgnore = True
Jeff Gastona7df0922021-06-02 17:39:47 -0400398 if addedIgnore:
399 source.addImport("org.junit.Ignore")
Jeff Gastona7df0922021-06-02 17:39:47 -0400400 source.save()
401 numUpdates += 1
Jeff Gastona7c77722023-11-02 15:15:59 -0400402 print("Made " + str(numUpdates) + " updates")
403
404def commit():
405 print("Generating git commit per OWNERS file")
406 os.chdir(supportRoot)
407 commitMessage = """Autogenerated suppression of test failures
Jeff Gastona7df0922021-06-02 17:39:47 -0400408
409This commit was created with the help of development/suppressFailingTests.py
Jeff Gastona7df0922021-06-02 17:39:47 -0400410"""
Jeff Gastona7c77722023-11-02 15:15:59 -0400411 shellRunner.run(["development/split_change_into_owners.sh", commitMessage])
Jeff Gastona7df0922021-06-02 17:39:47 -0400412
Jeff Gastona7df0922021-06-02 17:39:47 -0400413
414def main():
Jeff Gastona7c77722023-11-02 15:15:59 -0400415 global logger
416 arguments = parser.parse_args()
417 if arguments.v:
418 logger = PrintLogger()
419 else:
420 logger = DisabledLogger()
421 failures = getFailureData()
422 if len(failures) < 1:
423 print("Found 0 failures")
424 return
Jeff Gastona7df0922021-06-02 17:39:47 -0400425 locations = locate(failures)
426 disable(locations)
Jeff Gastona7c77722023-11-02 15:15:59 -0400427 commit()
Jeff Gastona7df0922021-06-02 17:39:47 -0400428
429if __name__ == "__main__":
430 main()
431