| # -*- coding: utf-8 -*- |
| import unittest |
| from typing import Dict, List |
| |
| from tools.stats import print_test_stats |
| from tools.stats.s3_stat_parser import ( |
| Commit, |
| Report, |
| ReportMetaMeta, |
| Status, |
| Version1Case, |
| Version1Report, |
| Version2Case, |
| Version2Report, |
| ) |
| |
| |
| def fakehash(char: str) -> str: |
| return char * 40 |
| |
| |
| def dummy_meta_meta() -> ReportMetaMeta: |
| return { |
| "build_pr": "", |
| "build_tag": "", |
| "build_sha1": "", |
| "build_base_commit": "", |
| "build_branch": "", |
| "build_job": "", |
| "build_workflow_id": "", |
| "build_start_time_epoch": "", |
| } |
| |
| |
| def makecase( |
| name: str, |
| seconds: float, |
| *, |
| errored: bool = False, |
| failed: bool = False, |
| skipped: bool = False, |
| ) -> Version1Case: |
| return { |
| "name": name, |
| "seconds": seconds, |
| "errored": errored, |
| "failed": failed, |
| "skipped": skipped, |
| } |
| |
| |
| def make_report_v1(tests: Dict[str, List[Version1Case]]) -> Version1Report: |
| suites = { |
| suite_name: { |
| "total_seconds": sum(case["seconds"] for case in cases), |
| "cases": cases, |
| } |
| for suite_name, cases in tests.items() |
| } |
| return { |
| **dummy_meta_meta(), # type: ignore[misc] |
| "total_seconds": sum(s["total_seconds"] for s in suites.values()), |
| "suites": suites, |
| } |
| |
| |
| def make_case_v2(seconds: float, status: Status = None) -> Version2Case: |
| return { |
| "seconds": seconds, |
| "status": status, |
| } |
| |
| |
| def make_report_v2( |
| tests: Dict[str, Dict[str, Dict[str, Version2Case]]] |
| ) -> Version2Report: |
| files = {} |
| for file_name, file_suites in tests.items(): |
| suites = { |
| suite_name: { |
| "total_seconds": sum(case["seconds"] for case in cases.values()), |
| "cases": cases, |
| } |
| for suite_name, cases in file_suites.items() |
| } |
| files[file_name] = { |
| "suites": suites, |
| "total_seconds": sum(suite["total_seconds"] for suite in suites.values()), |
| } |
| return { |
| **dummy_meta_meta(), # type: ignore[misc] |
| "format_version": 2, |
| "total_seconds": sum(s["total_seconds"] for s in files.values()), |
| "files": files, |
| } |
| |
| |
| maxDiff = None |
| |
| |
| class TestPrintTestStats(unittest.TestCase): |
| version1_report: Version1Report = make_report_v1( |
| { |
| # input ordering of the suites is ignored |
| "Grault": [ |
| # not printed: status same and time similar |
| makecase("test_grault0", 4.78, failed=True), |
| # status same, but time increased a lot |
| makecase("test_grault2", 1.473, errored=True), |
| ], |
| # individual tests times changed, not overall suite |
| "Qux": [ |
| # input ordering of the test cases is ignored |
| makecase("test_qux1", 0.001, skipped=True), |
| makecase("test_qux6", 0.002, skipped=True), |
| # time in bounds, but status changed |
| makecase("test_qux4", 7.158, failed=True), |
| # not printed because it's the same as before |
| makecase("test_qux7", 0.003, skipped=True), |
| makecase("test_qux5", 11.968), |
| makecase("test_qux3", 23.496), |
| ], |
| # new test suite |
| "Bar": [ |
| makecase("test_bar2", 3.742, failed=True), |
| makecase("test_bar1", 50.447), |
| ], |
| # overall suite time changed but no individual tests |
| "Norf": [ |
| makecase("test_norf1", 3), |
| makecase("test_norf2", 3), |
| makecase("test_norf3", 3), |
| makecase("test_norf4", 3), |
| ], |
| # suite doesn't show up if it doesn't change enough |
| "Foo": [ |
| makecase("test_foo1", 42), |
| makecase("test_foo2", 56), |
| ], |
| } |
| ) |
| |
| version2_report: Version2Report = make_report_v2( |
| { |
| "test_a": { |
| "Grault": { |
| "test_grault0": make_case_v2(4.78, "failed"), |
| "test_grault2": make_case_v2(1.473, "errored"), |
| }, |
| "Qux": { |
| "test_qux1": make_case_v2(0.001, "skipped"), |
| "test_qux6": make_case_v2(0.002, "skipped"), |
| "test_qux4": make_case_v2(7.158, "failed"), |
| "test_qux7": make_case_v2(0.003, "skipped"), |
| "test_qux8": make_case_v2(11.968), |
| "test_qux3": make_case_v2(23.496), |
| }, |
| }, |
| "test_b": { |
| "Bar": { |
| "test_bar2": make_case_v2(3.742, "failed"), |
| "test_bar1": make_case_v2(50.447), |
| }, |
| # overall suite time changed but no individual tests |
| "Norf": { |
| "test_norf1": make_case_v2(3), |
| "test_norf2": make_case_v2(3), |
| "test_norf3": make_case_v2(3), |
| "test_norf4": make_case_v2(3), |
| }, |
| }, |
| "test_c": { |
| "Foo": { |
| "test_foo1": make_case_v2(42), |
| "test_foo2": make_case_v2(56), |
| }, |
| }, |
| } |
| ) |
| |
| def test_simplify(self) -> None: |
| self.assertEqual( |
| { |
| "": { |
| "Bar": { |
| "test_bar1": {"seconds": 50.447, "status": None}, |
| "test_bar2": {"seconds": 3.742, "status": "failed"}, |
| }, |
| "Foo": { |
| "test_foo1": {"seconds": 42, "status": None}, |
| "test_foo2": {"seconds": 56, "status": None}, |
| }, |
| "Grault": { |
| "test_grault0": {"seconds": 4.78, "status": "failed"}, |
| "test_grault2": {"seconds": 1.473, "status": "errored"}, |
| }, |
| "Norf": { |
| "test_norf1": {"seconds": 3, "status": None}, |
| "test_norf3": {"seconds": 3, "status": None}, |
| "test_norf2": {"seconds": 3, "status": None}, |
| "test_norf4": {"seconds": 3, "status": None}, |
| }, |
| "Qux": { |
| "test_qux1": {"seconds": 0.001, "status": "skipped"}, |
| "test_qux3": {"seconds": 23.496, "status": None}, |
| "test_qux4": {"seconds": 7.158, "status": "failed"}, |
| "test_qux5": {"seconds": 11.968, "status": None}, |
| "test_qux6": {"seconds": 0.002, "status": "skipped"}, |
| "test_qux7": {"seconds": 0.003, "status": "skipped"}, |
| }, |
| }, |
| }, |
| print_test_stats.simplify(self.version1_report), |
| ) |
| |
| self.assertEqual( |
| { |
| "test_a": { |
| "Grault": { |
| "test_grault0": {"seconds": 4.78, "status": "failed"}, |
| "test_grault2": {"seconds": 1.473, "status": "errored"}, |
| }, |
| "Qux": { |
| "test_qux1": {"seconds": 0.001, "status": "skipped"}, |
| "test_qux3": {"seconds": 23.496, "status": None}, |
| "test_qux4": {"seconds": 7.158, "status": "failed"}, |
| "test_qux6": {"seconds": 0.002, "status": "skipped"}, |
| "test_qux7": {"seconds": 0.003, "status": "skipped"}, |
| "test_qux8": {"seconds": 11.968, "status": None}, |
| }, |
| }, |
| "test_b": { |
| "Bar": { |
| "test_bar1": {"seconds": 50.447, "status": None}, |
| "test_bar2": {"seconds": 3.742, "status": "failed"}, |
| }, |
| "Norf": { |
| "test_norf1": {"seconds": 3, "status": None}, |
| "test_norf2": {"seconds": 3, "status": None}, |
| "test_norf3": {"seconds": 3, "status": None}, |
| "test_norf4": {"seconds": 3, "status": None}, |
| }, |
| }, |
| "test_c": { |
| "Foo": { |
| "test_foo1": {"seconds": 42, "status": None}, |
| "test_foo2": {"seconds": 56, "status": None}, |
| }, |
| }, |
| }, |
| print_test_stats.simplify(self.version2_report), |
| ) |
| |
| def test_analysis(self) -> None: |
| head_report = self.version1_report |
| |
| base_reports: Dict[Commit, List[Report]] = { |
| # bbbb has no reports, so base is cccc instead |
| fakehash("b"): [], |
| fakehash("c"): [ |
| make_report_v1( |
| { |
| "Baz": [ |
| makecase("test_baz2", 13.605), |
| # no recent suites have & skip this test |
| makecase("test_baz1", 0.004, skipped=True), |
| ], |
| "Foo": [ |
| makecase("test_foo1", 43), |
| # test added since dddd |
| makecase("test_foo2", 57), |
| ], |
| "Grault": [ |
| makecase("test_grault0", 4.88, failed=True), |
| makecase("test_grault1", 11.967, failed=True), |
| makecase("test_grault2", 0.395, errored=True), |
| makecase("test_grault3", 30.460), |
| ], |
| "Norf": [ |
| makecase("test_norf1", 2), |
| makecase("test_norf2", 2), |
| makecase("test_norf3", 2), |
| makecase("test_norf4", 2), |
| ], |
| "Qux": [ |
| makecase("test_qux3", 4.978, errored=True), |
| makecase("test_qux7", 0.002, skipped=True), |
| makecase("test_qux2", 5.618), |
| makecase("test_qux4", 7.766, errored=True), |
| makecase("test_qux6", 23.589, failed=True), |
| ], |
| } |
| ), |
| ], |
| fakehash("d"): [ |
| make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo1", 40), |
| # removed in cccc |
| makecase("test_foo3", 17), |
| ], |
| "Baz": [ |
| # not skipped, so not included in stdev |
| makecase("test_baz1", 3.14), |
| ], |
| "Qux": [ |
| makecase("test_qux7", 0.004, skipped=True), |
| makecase("test_qux2", 6.02), |
| makecase("test_qux4", 20.932), |
| ], |
| "Norf": [ |
| makecase("test_norf1", 3), |
| makecase("test_norf2", 3), |
| makecase("test_norf3", 3), |
| makecase("test_norf4", 3), |
| ], |
| "Grault": [ |
| makecase("test_grault0", 5, failed=True), |
| makecase("test_grault1", 14.325, failed=True), |
| makecase("test_grault2", 0.31, errored=True), |
| ], |
| } |
| ), |
| ], |
| fakehash("e"): [], |
| fakehash("f"): [ |
| make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo3", 24), |
| makecase("test_foo1", 43), |
| ], |
| "Baz": [ |
| makecase("test_baz2", 16.857), |
| ], |
| "Qux": [ |
| makecase("test_qux2", 6.422), |
| makecase("test_qux4", 6.382, errored=True), |
| ], |
| "Norf": [ |
| makecase("test_norf1", 0.9), |
| makecase("test_norf3", 0.9), |
| makecase("test_norf2", 0.9), |
| makecase("test_norf4", 0.9), |
| ], |
| "Grault": [ |
| makecase("test_grault0", 4.7, failed=True), |
| makecase("test_grault1", 13.146, failed=True), |
| makecase("test_grault2", 0.48, errored=True), |
| ], |
| } |
| ), |
| ], |
| } |
| |
| simpler_head = print_test_stats.simplify(head_report) |
| simpler_base = {} |
| for commit, reports in base_reports.items(): |
| simpler_base[commit] = [print_test_stats.simplify(r) for r in reports] |
| analysis = print_test_stats.analyze( |
| head_report=simpler_head, |
| base_reports=simpler_base, |
| ) |
| |
| self.assertEqual( |
| """\ |
| |
| - class Baz: |
| - # was 15.23s ± 2.30s |
| - |
| - def test_baz1: ... |
| - # was 0.004s (skipped) |
| - |
| - def test_baz2: ... |
| - # was 15.231s ± 2.300s |
| |
| |
| class Grault: |
| # was 48.86s ± 1.19s |
| # now 6.25s |
| |
| - def test_grault1: ... |
| - # was 13.146s ± 1.179s (failed) |
| |
| - def test_grault3: ... |
| - # was 30.460s |
| |
| |
| class Qux: |
| # was 41.66s ± 1.06s |
| # now 42.63s |
| |
| - def test_qux2: ... |
| - # was 6.020s ± 0.402s |
| |
| ! def test_qux3: ... |
| ! # was 4.978s (errored) |
| ! # now 23.496s |
| |
| ! def test_qux4: ... |
| ! # was 7.074s ± 0.979s (errored) |
| ! # now 7.158s (failed) |
| |
| ! def test_qux6: ... |
| ! # was 23.589s (failed) |
| ! # now 0.002s (skipped) |
| |
| + def test_qux1: ... |
| + # now 0.001s (skipped) |
| |
| + def test_qux5: ... |
| + # now 11.968s |
| |
| |
| + class Bar: |
| + # now 54.19s |
| + |
| + def test_bar1: ... |
| + # now 50.447s |
| + |
| + def test_bar2: ... |
| + # now 3.742s (failed) |
| |
| """, |
| print_test_stats.anomalies(analysis), |
| ) |
| |
| def test_graph(self) -> None: |
| # HEAD is on master |
| self.assertEqual( |
| """\ |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| * aaaaaaaaaa (HEAD) total time 502.99s |
| * bbbbbbbbbb (base) 1 report, total time 47.84s |
| * cccccccccc 1 report, total time 332.50s |
| * dddddddddd 0 reports |
| | |
| : |
| """, |
| print_test_stats.graph( |
| head_sha=fakehash("a"), |
| head_seconds=502.99, |
| base_seconds={ |
| fakehash("b"): [47.84], |
| fakehash("c"): [332.50], |
| fakehash("d"): [], |
| }, |
| on_master=True, |
| ), |
| ) |
| |
| self.assertEqual( |
| """\ |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 9988.77s |
| |/ |
| * bbbbbbbbbb (base) 121 reports, total time 7654.32s ± 55.55s |
| * cccccccccc 20 reports, total time 5555.55s ± 253.19s |
| * dddddddddd 1 report, total time 1234.56s |
| | |
| : |
| """, |
| print_test_stats.graph( |
| head_sha=fakehash("a"), |
| head_seconds=9988.77, |
| base_seconds={ |
| fakehash("b"): [7598.77] * 60 + [7654.32] + [7709.87] * 60, |
| fakehash("c"): [5308.77] * 10 + [5802.33] * 10, |
| fakehash("d"): [1234.56], |
| }, |
| on_master=False, |
| ), |
| ) |
| |
| self.assertEqual( |
| """\ |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 25.52s |
| | | |
| | : (5 commits) |
| |/ |
| * bbbbbbbbbb 0 reports |
| * cccccccccc 0 reports |
| * dddddddddd (base) 15 reports, total time 58.92s ± 25.82s |
| | |
| : |
| """, |
| print_test_stats.graph( |
| head_sha=fakehash("a"), |
| head_seconds=25.52, |
| base_seconds={ |
| fakehash("b"): [], |
| fakehash("c"): [], |
| fakehash("d"): [52.25] * 14 + [152.26], |
| }, |
| on_master=False, |
| ancestry_path=5, |
| ), |
| ) |
| |
| self.assertEqual( |
| """\ |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 0.08s |
| |/| |
| | : (1 commit) |
| | |
| * bbbbbbbbbb 0 reports |
| * cccccccccc (base) 1 report, total time 0.09s |
| * dddddddddd 3 reports, total time 0.10s ± 0.05s |
| | |
| : |
| """, |
| print_test_stats.graph( |
| head_sha=fakehash("a"), |
| head_seconds=0.08, |
| base_seconds={ |
| fakehash("b"): [], |
| fakehash("c"): [0.09], |
| fakehash("d"): [0.05, 0.10, 0.15], |
| }, |
| on_master=False, |
| other_ancestors=1, |
| ), |
| ) |
| |
| self.assertEqual( |
| """\ |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 5.98s |
| | | |
| | : (1 commit) |
| |/| |
| | : (7 commits) |
| | |
| * bbbbbbbbbb (base) 2 reports, total time 6.02s ± 1.71s |
| * cccccccccc 0 reports |
| * dddddddddd 10 reports, total time 5.84s ± 0.92s |
| | |
| : |
| """, |
| print_test_stats.graph( |
| head_sha=fakehash("a"), |
| head_seconds=5.98, |
| base_seconds={ |
| fakehash("b"): [4.81, 7.23], |
| fakehash("c"): [], |
| fakehash("d"): [4.97] * 5 + [6.71] * 5, |
| }, |
| on_master=False, |
| ancestry_path=1, |
| other_ancestors=7, |
| ), |
| ) |
| |
| def test_regression_info(self) -> None: |
| self.assertEqual( |
| """\ |
| ----- Historic stats comparison result ------ |
| |
| job: foo_job |
| commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa |
| |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 3.02s |
| |/ |
| * bbbbbbbbbb (base) 1 report, total time 41.00s |
| * cccccccccc 1 report, total time 43.00s |
| | |
| : |
| |
| Removed (across 1 suite) 1 test, totaling - 1.00s |
| Modified (across 1 suite) 1 test, totaling - 41.48s ± 2.12s |
| Added (across 1 suite) 1 test, totaling + 3.00s |
| """, |
| print_test_stats.regression_info( |
| head_sha=fakehash("a"), |
| head_report=make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo", 0.02, skipped=True), |
| makecase("test_baz", 3), |
| ] |
| } |
| ), |
| base_reports={ |
| fakehash("b"): [ |
| make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo", 40), |
| makecase("test_bar", 1), |
| ], |
| } |
| ), |
| ], |
| fakehash("c"): [ |
| make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo", 43), |
| ], |
| } |
| ), |
| ], |
| }, |
| job_name="foo_job", |
| on_master=False, |
| ancestry_path=0, |
| other_ancestors=0, |
| ), |
| ) |
| |
| def test_regression_info_new_job(self) -> None: |
| self.assertEqual( |
| """\ |
| ----- Historic stats comparison result ------ |
| |
| job: foo_job |
| commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa |
| |
| Commit graph (base is most recent master ancestor with at least one S3 report): |
| |
| : (master) |
| | |
| | * aaaaaaaaaa (HEAD) total time 3.02s |
| | | |
| | : (3 commits) |
| |/| |
| | : (2 commits) |
| | |
| * bbbbbbbbbb 0 reports |
| * cccccccccc 0 reports |
| | |
| : |
| |
| Removed (across 0 suites) 0 tests, totaling 0.00s |
| Modified (across 0 suites) 0 tests, totaling 0.00s |
| Added (across 1 suite) 2 tests, totaling + 3.02s |
| """, |
| print_test_stats.regression_info( |
| head_sha=fakehash("a"), |
| head_report=make_report_v1( |
| { |
| "Foo": [ |
| makecase("test_foo", 0.02, skipped=True), |
| makecase("test_baz", 3), |
| ] |
| } |
| ), |
| base_reports={ |
| fakehash("b"): [], |
| fakehash("c"): [], |
| }, |
| job_name="foo_job", |
| on_master=False, |
| ancestry_path=3, |
| other_ancestors=2, |
| ), |
| ) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |
