Merge "compare_cts_reports: add the `--ignore-abi` option" into main am: c60d41238b am: 9ac8bae07d

Original change: https://android-review.googlesource.com/c/platform/development/+/2862616

Change-Id: I691d72040bae59a78d62f50c25eaed48461ca9de
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

4 files changed, 25 insertions, 13 deletions

diff --git a/tools/compare_cts_reports/README.md b/tools/compare_cts_reports/README.md
index 99cd3a1ad..f7c5cd2f9 100644
--- a/tools/compare_cts_reports/README.md
+++ b/tools/compare_cts_reports/README.md
@@ -20,17 +20,19 @@ The `-d` flag specifies the directory in which the information files will be sto
 ## aggregate_cts_reports.py
 ### usage
 ```
-./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR
+./aggregate_cts_reports.py -r REPORT [REPORT ...] -d OUTPUT_DIR [--ignore-abi]
 ```
 
 The `-r` flag can be followed by one or more reports.
 
 The `-d` flag has the same behavior as `parse_cts_report.py`.
 
+`--ignore-abi` is a boolean flag. If users specify this flag, tests ABI would be ignored while doing the aggregation. It means that two tests would be considered as the same one as long as they have the same module_name#class_name.test_name.
+
 ## compare_cts_reports.py
 ### usage
 ```
-./compare_cts_reports.py [-h] [-r CTS_REPORTS [CTS_REPORTS ...]] [-f CTS_REPORTS] --mode {1,2,n} --output-dir OUTPUT_DIR [--csv CSV] [--output-files]
+./compare_cts_reports.py [-h] [-r CTS_REPORTS [CTS_REPORTS ...]] [-f CTS_REPORTS] --mode {1,2,n} --output-dir OUTPUT_DIR [--csv CSV] [--output-files] [--ignore-abi]
 ```
 
 One `-r` flag is followed by a group of report files that you want to aggregate.
@@ -41,9 +43,11 @@ The `-m` flag has to be specified: `1` for one-way mode, `2` for two-way mode, a
 
 The `-d` flag has to be specified. Behavior same as `parse_cts_report.py`.
 
+`--csv` allows users to specify the file name of the comparison result. The default value is `diff.csv`.
+
 `--output-files/-o` is a boolean flag. If users specify this flag, the parsed results of reports after flags `-r` will be outputed into the information files.
 
-`--csv` allows users to specify the file name of the comparison result. The default value is `diff.csv`.
+`--ignore-abi` is a boolean flag, which has the same behavior as `aggregate_cts_reports.py`.
 
 ### modes
 #### One-way Comparison
@@ -64,4 +68,4 @@ N-way comparison mode is used to show a summary of several test reports.
 ./test_parse_cts_report.py
 ./test_aggregate_cts_reports.py
 ./test_compare_cts_reports.py
-```
\ No newline at end of file
+```
diff --git a/tools/compare_cts_reports/aggregate_cts_reports.py b/tools/compare_cts_reports/aggregate_cts_reports.py
index 82d754e06..f2db373e2 100755
--- a/tools/compare_cts_reports/aggregate_cts_reports.py
+++ b/tools/compare_cts_reports/aggregate_cts_reports.py
@@ -29,7 +29,7 @@ import zipfile
 import parse_cts_report
 
 
-def aggregate_cts_reports(report_files):
+def aggregate_cts_reports(report_files, ignore_abi=False):
   """Aggregate all report files and produce information files to output_dir.
 
   If the results of the same test are different in two reports, choose the one
@@ -38,6 +38,9 @@ def aggregate_cts_reports(report_files):
 
   Args:
     report_files: A list of paths to cts reports.
+    ignore_abi: Ignore the tests ABI when doing the aggregation, which means
+                that tests would be considered as the same one as long as they
+                have the same module_name#class_name.test_name.
 
   Raises:
     UserWarning: Report files not compatible.
@@ -48,7 +51,7 @@ def aggregate_cts_reports(report_files):
 
   first_report_file = report_files[0]
 
-  report = parse_cts_report.parse_report_file(first_report_file)
+  report = parse_cts_report.parse_report_file(first_report_file, ignore_abi)
 
   with tempfile.TemporaryDirectory() as temp_dir:
 
@@ -64,7 +67,7 @@ def aggregate_cts_reports(report_files):
         msg = (f'{report_file} is incompatible to {first_report_file}.')
         raise UserWarning(msg)
 
-      report.read_test_result_xml(xml_path)
+      report.read_test_result_xml(xml_path, ignore_abi)
 
   return report
 
@@ -77,6 +80,8 @@ def main():
                             'be a zip archive or a xml file.'))
   parser.add_argument('-d', '--output-dir', required=True,
                       help=('Path to the directory to store output files.'))
+  parser.add_argument('--ignore-abi', action='store_true',
+                      help='Ignore the tests ABI while aggregating reports.')
 
   args = parser.parse_args()
 
@@ -86,7 +91,7 @@ def main():
   if not os.path.exists(output_dir):
     raise FileNotFoundError(f'Output directory {output_dir} does not exist.')
 
-  report = aggregate_cts_reports(report_files)
+  report = aggregate_cts_reports(report_files, args.ignore_abi)
   report.output_files(output_dir)
 
 
diff --git a/tools/compare_cts_reports/compare_cts_reports.py b/tools/compare_cts_reports/compare_cts_reports.py
index 006f63638..168af2058 100755
--- a/tools/compare_cts_reports/compare_cts_reports.py
+++ b/tools/compare_cts_reports/compare_cts_reports.py
@@ -270,6 +270,8 @@ def main():
   parser.add_argument('--csv', default='diff.csv', help='Path to csv output.')
   parser.add_argument('--output-files', '-o', action='store_true',
                       help='Output parsed csv files.')
+  parser.add_argument('--ignore-abi', action='store_true',
+                      help='Ignore the tests ABI while comparing.')
 
   args = parser.parse_args()
 
@@ -287,10 +289,11 @@ def main():
   diff_csv = os.path.join(output_dir, args.csv)
 
   ctsreports = []
+  ignore_abi = args.ignore_abi
   for i, report_path in enumerate(reports):
     is_report_files = isinstance(report_path, list)
     report = (
-        aggregate_cts_reports.aggregate_cts_reports(report_path)
+        aggregate_cts_reports.aggregate_cts_reports(report_path, ignore_abi)
         if is_report_files  # path(s) come from --report flag
         else load_parsed_report(report_path)
     )
diff --git a/tools/compare_cts_reports/parse_cts_report.py b/tools/compare_cts_reports/parse_cts_report.py
index f6a082de4..d045c21d7 100755
--- a/tools/compare_cts_reports/parse_cts_report.py
+++ b/tools/compare_cts_reports/parse_cts_report.py
@@ -126,7 +126,7 @@ class CtsReport:
       summary.counter[previous] -= 1
       summary.counter[test_status] += 1
 
-  def read_test_result_xml(self, test_result_path):
+  def read_test_result_xml(self, test_result_path, ignore_abi=False):
     """Read the result from test_result.xml into a CtsReport object."""
 
     tree = ET.parse(test_result_path)
@@ -134,7 +134,7 @@ class CtsReport:
 
     for module in root.iter('Module'):
       module_name = module.attrib['name']
-      abi = module.attrib['abi']
+      abi = 'ignored' if ignore_abi else module.attrib['abi']
 
       for testcase in module.iter('TestCase'):
         class_name = testcase.attrib['name']
@@ -341,7 +341,7 @@ def extract_test_result_from_zip(zip_file_path, dest_dir):
   return extracted
 
 
-def parse_report_file(report_file):
+def parse_report_file(report_file, ignore_abi=False):
   """Turn one cts report into a CtsReport object."""
 
   with tempfile.TemporaryDirectory() as temp_dir:
@@ -355,7 +355,7 @@ def parse_report_file(report_file):
     print_test_info(test_info)
 
     report = CtsReport(test_info)
-    report.read_test_result_xml(xml_path)
+    report.read_test_result_xml(xml_path, ignore_abi)
 
   return report