Write the log analysis report in a proto file.

Read the keywords from a yaml config file.
Write the output as a proto file.

Bug: 262259622
Test: Manually.
Change-Id: Ice59125e6e89e87c91597f2a34d3a0204b964c63

4 files changed, 185 insertions, 30 deletions

diff --git a/perf_tools/Android.bp b/perf_tools/Android.bp
new file mode 100644
index 00000000..26b93511
--- /dev/null
+++ b/perf_tools/Android.bp
@@ -0,0 +1,38 @@
+//
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+    // See: http://go/android-license-faq
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+python_binary_host {
+    name: "progress_report",
+    srcs: ["progress_report.py"],
+    libs: [ "report_proto",],
+    version: {
+    	     py3: { embedded_launcher: true },
+    },
+    main: "progress_report.py",
+}
+
+python_library_host {
+    name: "report_proto",
+    srcs: ["*.proto"],
+    proto: {
+        canonical_path_from_root: false,
+    },
+}
diff --git a/perf_tools/config.yaml b/perf_tools/config.yaml
new file mode 100644
index 00000000..4372ce20
--- /dev/null
+++ b/perf_tools/config.yaml
@@ -0,0 +1,15 @@
+# This file is used to store the keywords to be extracted
+# from the logcat.
+---
+- boot_progress_start
+- boot_progress_preload_start
+- boot_progress_preload_end
+- boot_progress_system_run
+- boot_progress_pms_start
+- boot_progress_pms_system_scan_start
+- boot_progress_pms_data_scan_start
+- boot_progress_pms_scan_end
+- boot_progress_pms_ready
+- boot_progress_ams_ready
+- boot_progress_enable_screen
+- car_helper_boot_phase
diff --git a/perf_tools/progress_report.py b/perf_tools/progress_report.py
index 9d1d8afc..f6a1c430 100644..100755
--- a/perf_tools/progress_report.py
+++ b/perf_tools/progress_report.py
@@ -1,8 +1,57 @@
-import sys
-import os
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
 from datetime import datetime
-
-#find boot_progress_start line and boot_progress_enable_screen find the time difference
+import yaml
+import os
+import report_pb2
+import sys
+import traceback
+
+# Usage: python3 progress_report.py --logcat logcat.txt --config config.yaml --output_dir report_dir
+#
+# logcat.txt should contain the "boot_progress_start" and "boot_progress_enable_screen"".
+# config.yaml contains all the keywords to be extracted.
+# report_dir will contain three generated files:
+#
+# timestamp_log.txt: contains the same content as logcat.txt, but the timestamp is replaced
+# with relative time with boot_progress_start time.
+#
+# report_proto.txt: contains the report for the events related to the keywords.
+#
+# report.txt: contains logcat messages corresponding to the events captured in report_proto.txt
+
+def init_arguments():
+    parser = argparse.ArgumentParser(
+        prog = 'progrocess_report.py',
+        description='Extract timing information and generate a report.')
+    parser.add_argument(
+        '--logcat', type=str, required=True,
+        help = 'logcat file name')
+    parser.add_argument(
+        '--config', type=str, required=True,
+        help = 'configuration file for keywords')
+    parser.add_argument(
+        '--output_dir', type= str, required=True,
+        help = 'directory name to store the generated files')
+    return parser.parse_args()
+
+# Find boot_progress_start line and boot_progress_enable_screen find the time difference
 # return the start time string
 def find_boot_progress_start_end(fp):
     start = ""
@@ -13,12 +62,18 @@ def find_boot_progress_start_end(fp):
         if "boot_progress_enable_screen" in line and len(start):
             end = line
             break
+
+    missing_error = ""
+    if start == "":
+        missing_error = "******logcat file missing boot_progress_start\n"
+    elif end == "":
+        missing_error +=  "******logcat file missing boot_progress_end "
+    if missing_error != "":
+        sys.exit("Missing required message in the logcat:\n" + missing_error)
     return [start, end]
 
+# TODO(b/262259622): passing a tuple of (startDate, endDate)
 def replace_timestamp_abs(line, timestamp_str, date_time_obj0):
-    if line[:5] != timestamp_str[:5]:
-        return line
-
     index = line.find(" ", 6)
     if index <= 0:
         return line
@@ -45,34 +100,60 @@ def in_time_range(start, end, line):
 
     return False
 
-def write_to_new_file(fp, output_fp, summary_fp, timestamps):
+# Here is an example of event we would like extract:
+# 09-15 16:04:15.655  root   991   991 I boot_progress_preload_start: 5440
+# for each event, it is a tuple of(timestamp, event_name, timing)
+def extract_event(line, keywords):
+    words = line.split(" ")
+    for keyword in keywords:
+        if keyword in words[-2]:
+            return (words[0], words[-2], words[-1])
+    return ()
+
+def write_to_new_file(timestamps, keywords, logcat_fp, timestamp_fixed_logcat_fp, report_fp,
+                      report_proto_fp):
     start_timestamp_obj = datetime.strptime(timestamps[0][:18], '%m-%d %H:%M:%S.%f')
     end_timestamp_obj = datetime.strptime(timestamps[1][:18], '%m-%d %H:%M:%S.%f')
+    report = report_pb2.Report()
+    for line in logcat_fp:
+        ts_fixed_line = replace_timestamp_abs(line, timestamps[0][:18], start_timestamp_obj)
+        timestamp_fixed_logcat_fp.write(ts_fixed_line)
+        if in_time_range(start_timestamp_obj, end_timestamp_obj, line):
+            event = extract_event(ts_fixed_line, keywords)
+            if len(event) == 0:
+                continue
+
+            report_fp.write(ts_fixed_line)
+            record = report.record.add()
+            record.timestamp = event[0]
+            record.event = event[1]
+            record.timing = int(event[2])
+    report_proto_fp.write(str(report))
 
-    for line in fp:
-        newline = replace_timestamp_abs(line, timestamps[0][:18], start_timestamp_obj)
-        output_fp.write(newline)
-        if "boot_progress_" in newline and in_time_range(start_timestamp_obj, end_timestamp_obj, line):
-            summary_fp.write(newline)
+def main():
+    args = init_arguments()
 
+    keywords = []
+    with open(args.config, 'r') as file:
+        keywords = yaml.safe_load(file)
 
-def main():
-    filepath = sys.argv[1]
-    if not os.path.isfile(filepath):
-        print("File path {} does not exist. Exiting...".format(filepath))
-        sys.exit()
-
-    output_fp = open(sys.argv[2], 'w')
-    summary_fp = open(sys.argv[3], 'w')
-
-    with open(filepath, 'r', errors = 'ignore') as fp:
-        timestamps = find_boot_progress_start_end(fp)
-        fp.seek(0)
-        write_to_new_file(fp, output_fp, summary_fp, timestamps)
-
-    fp.close()
-    output_fp.close()
-    summary_fp.close()
+    if not os.path.isdir(args.output_dir):
+        os.mkdir(args.output_dir)
+    timestamp_fixed_logcat_fp = open(os.path.join(args.output_dir, "timestamp_fixed_log.txt"), 'w')
+    report_fp = open(os.path.join(args.output_dir, "report.txt"), 'w')
+    report_proto_fp = open(os.path.join(args.output_dir,  "report_proto.txt"), 'w')
+    try:
+        with open(args.logcat, 'r', errors = 'ignore') as logcat_fp:
+            timestamps = find_boot_progress_start_end(logcat_fp)
+            logcat_fp.seek(0)
+            write_to_new_file(timestamps, keywords, logcat_fp, timestamp_fixed_logcat_fp, report_fp, report_proto_fp)
+    except Exception as e:
+        traceresult = traceback.format_exc()
+        print("Caught an exception: {}".format(traceback.format_exc()))
+
+    timestamp_fixed_logcat_fp.close()
+    report_fp.close()
+    report_proto_fp.close()
 
 if __name__ == '__main__':
     main()
diff --git a/perf_tools/report.proto b/perf_tools/report.proto
new file mode 100644
index 00000000..fb9e839b
--- /dev/null
+++ b/perf_tools/report.proto
@@ -0,0 +1,21 @@
+syntax = "proto2";
+
+package report;
+
+message Keyword {
+  required string value = 1;
+}
+
+message Keywords {
+  repeated Keyword keyword = 1;
+}
+
+message Record {
+  required string timestamp = 1;
+  required string event = 2;
+  required int64 timing = 3;
+}
+
+message Report {
+  repeated Record record = 1;
+}