perf_metrics: Add long RT runnable metrics

Bug: 275806676
Change-Id: I2f319baea00f2514983b5f4f8024ba4dbc826889

4 files changed, 304 insertions, 41 deletions

diff --git a/drivers/soc/google/vh/include/sched.h b/drivers/soc/google/vh/include/sched.h
index 22e804de9..9af8bd519 100644
--- a/drivers/soc/google/vh/include/sched.h
+++ b/drivers/soc/google/vh/include/sched.h
@@ -72,6 +72,7 @@ struct vendor_task_struct {
 	struct vendor_binder_task_struct binder_task;
 	/* parameters for RT inheritance */
 	unsigned int uclamp_pi[UCLAMP_CNT];
+	u64 runnable_start_ns;
 };
 
 ANDROID_VENDOR_CHECK_SIZE_ALIGN(u64 android_vendor_data1[64], struct vendor_task_struct t);
diff --git a/drivers/soc/google/vh/kernel/metrics/perf_metrics.c b/drivers/soc/google/vh/kernel/metrics/perf_metrics.c
index 78851f324..f50e81291 100644
--- a/drivers/soc/google/vh/kernel/metrics/perf_metrics.c
+++ b/drivers/soc/google/vh/kernel/metrics/perf_metrics.c
@@ -21,17 +21,151 @@
 #include <trace/events/power.h>
 #include <trace/events/irq.h>
 #include <trace/hooks/suspend.h>
+#include <trace/hooks/sched.h>
 #include "perf_metrics.h"
 #include <trace/hooks/systrace.h>
+#include "../../include/sched.h"
+#include <kernel/sched/sched.h>
+
+struct irq_storm_data {
+	atomic64_t storm_count;
+	s64 max_storm_count;
+	atomic64_t irq_storm_start;
+};
+
+struct resume_latency {
+	u64 resume_start;
+	u64 resume_end;
+	spinlock_t resume_latency_stat_lock;
+	s64 resume_count[RESUME_LATENCY_ARR_SIZE];
+	u64 resume_latency_max_ms;
+	u64 resume_latency_sum_ms;
+	u64 resume_latency_threshold;
+	bool display_warning;
+};
+
+struct long_irq {
+	ktime_t softirq_start[CONFIG_VH_SCHED_CPU_NR][NR_SOFTIRQS];
+	ktime_t irq_start[CONFIG_VH_SCHED_CPU_NR][MAX_IRQ_NUM];
+	atomic64_t long_softirq_count;
+	atomic64_t long_irq_count;
+	atomic64_t long_softirq_count_arr[CONFIG_VH_SCHED_CPU_NR];
+	atomic64_t long_irq_count_arr[CONFIG_VH_SCHED_CPU_NR];
+	s64 long_softirq_arr[NR_SOFTIRQS];
+	s64 long_irq_arr[MAX_IRQ_NUM];
+	struct irq_storm_data irq_storms[MAX_IRQ_NUM];
+	s64 long_softirq_threshold;
+	s64 long_irq_threshold;
+	s64 irq_storm_threshold_us;
+	bool display_warning;
+};
+
+struct rt_runnable {
+	u64 latency;
+	char *comm;
+	char *prev_comm;
+	pid_t pid;
+};
+
+struct top_rt_runnable {
+	struct rt_runnable rt_runnable[RT_RUNNABLE_ARR_SIZE];
+	int min_idx;
+	atomic64_t count;
+};
 
 struct irq_entry {
 	int irq_num;
 	s64 latency;
 	s64 max_storm_count;
 };
+
 static struct resume_latency resume_latency_stats;
 static struct long_irq long_irq_stat;
 
+static DEFINE_PER_CPU(struct top_rt_runnable, top_rt_runnable);
+static DEFINE_PER_CPU(spinlock_t, rt_runnable_lock);
+unsigned long long_rt_runnable_threshold_ns = 1500000UL;
+
+/*********************************************************************
+ *                          HELPER FUNCTIONS                         *
+ *********************************************************************/
+
+static bool is_top_latency(u64 latency)
+{
+	struct top_rt_runnable trr;
+	int cpu = raw_smp_processor_id();
+
+	spin_lock(&per_cpu(rt_runnable_lock, cpu));
+	trr = per_cpu(top_rt_runnable, cpu);
+	spin_unlock(&per_cpu(rt_runnable_lock, cpu));
+
+	return latency > trr.rt_runnable[trr.min_idx].latency;
+}
+
+static void update_min_latency(struct task_struct *prev, struct task_struct *next, u64 latency)
+{
+	u64 min_latency = -1LL;
+	struct rt_runnable *rt_runnable;
+	struct top_rt_runnable *trr;
+	int min_idx, i, cpu;
+	bool in_arr = false;
+
+	cpu = raw_smp_processor_id();
+	spin_lock(&per_cpu(rt_runnable_lock, cpu));
+	trr = this_cpu_ptr(&top_rt_runnable);
+
+	atomic64_inc(&(trr->count));
+
+	/* Search if the pid is already in the top_runnable, if so, update it */
+	for (i = 0; i < RT_RUNNABLE_ARR_SIZE; i++) {
+		rt_runnable =  &trr->rt_runnable[i];
+		if (rt_runnable->pid == next->pid) {
+			in_arr = true;
+			rt_runnable->latency = latency;
+			break;
+		}
+	}
+	if (!in_arr){
+		rt_runnable = &trr->rt_runnable[trr->min_idx];
+		rt_runnable->latency = latency;
+		rt_runnable->comm = kstrdup(next->comm, GFP_KERNEL);
+		rt_runnable->prev_comm = kstrdup(prev->comm, GFP_KERNEL);
+		rt_runnable->pid = next->pid;
+	}
+
+	/* Update min entry among top_rt_runnable */
+	for (i = 0; i < RT_RUNNABLE_ARR_SIZE; i++) {
+		rt_runnable =  &trr->rt_runnable[i];
+		if (rt_runnable->latency < min_latency) {
+			min_latency = rt_runnable->latency;
+			min_idx = i;
+		}
+	}
+
+	trr->min_idx = min_idx;
+	spin_unlock(&per_cpu(rt_runnable_lock, cpu));
+}
+
+static int irq_latency_cmp(const void *a, const void *b)
+{
+	return ((struct irq_entry *)b)->latency - ((struct irq_entry *)a)->latency;
+}
+
+static int irq_storm_count_cmp(const void *a, const void *b)
+{
+	return ((struct irq_entry *)b)->max_storm_count - ((struct irq_entry *)a)->max_storm_count;
+}
+
+static int runnable_latency_cmp(const void *a, const void *b)
+{
+	if (((struct rt_runnable *)a)->latency < ((struct rt_runnable *)b)->latency)
+		return 1;
+	else if (((struct rt_runnable *)a)->latency > ((struct rt_runnable *)b)->latency)
+		return -1;
+	else
+		return 0;
+}
+
 /*********************************************************************
  *                          SYSTEM TRACE
  *********************************************************************/
@@ -204,18 +338,47 @@ static void hook_irq_end(void *data, int irq, struct irqaction *action, int ret)
 						curr_max_irq, irq_usec) != curr_max_irq);
 }
 
-/*********************************************************************
- *                          HELPER FUNCTIONS                         *
- *********************************************************************/
-
-static int irq_latency_cmp(const void *a, const void *b)
+void vh_sched_wakeup_pixel_mod(void *data, struct task_struct *p)
 {
-	return ((struct irq_entry *)b)->latency - ((struct irq_entry *)a)->latency;
+	struct vendor_task_struct *vp;
+
+	if (!rt_task(p))
+		return;
+	vp = get_vendor_task_struct(p);
+	vp->runnable_start_ns = sched_clock();
 }
 
-static int irq_storm_count_cmp(const void *a, const void *b)
+void vh_sched_switch_pixel_mod(void *data, bool preempt,
+		struct task_struct *prev,
+		struct task_struct *next)
 {
-	return ((struct irq_entry *)b)->max_storm_count - ((struct irq_entry *)a)->max_storm_count;
+	struct vendor_task_struct *vnext, *vprev;
+	u64 now, runnable_delta;
+
+	now = sched_clock();
+	vprev = get_vendor_task_struct(prev);
+
+	/*
+	 * Update previous task's runnable_start_ns if it is in TASK_RUNNING state,
+	 * which means it remains in rq. Otherwise, invalidate runnable_start_ns,
+	 * given task is dequeued.
+	 */
+	if (task_is_running(prev) && rt_task(prev))
+		vprev->runnable_start_ns = now;
+	else
+		vprev->runnable_start_ns = -1;
+
+	vnext = get_vendor_task_struct(next);
+	if (!rt_task(next) || vnext->runnable_start_ns > now)
+		return;
+
+	runnable_delta = now - vnext->runnable_start_ns;
+	if (runnable_delta < long_rt_runnable_threshold_ns ||
+		!is_top_latency(runnable_delta))
+		return;
+
+	update_min_latency(prev, next, runnable_delta);
+
 }
 
 /*******************************************************************
@@ -504,6 +667,7 @@ static ssize_t irq_display_warning_store(struct kobject *kobj,
 	}
 	return count;
 }
+
 static ssize_t irq_stats_reset_store(struct kobject *kobj,
 					  struct kobj_attribute *attr,
 					  const char *buf,
@@ -519,6 +683,82 @@ static ssize_t irq_stats_reset_store(struct kobject *kobj,
 	atomic64_set(&(long_irq_stat.long_softirq_count), 0);
 	return count;
 }
+
+static ssize_t long_runnable_metrics_show(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 char *buf)
+{
+	int cpu, i;
+	ssize_t count = 0;
+	struct top_rt_runnable trr;
+	struct rt_runnable long_rt_runnable;
+	struct rt_runnable sorted_trr[RT_RUNNABLE_ARR_SIZE];
+
+	for_each_possible_cpu(cpu) {
+		count += sysfs_emit_at(buf, count, "cpu %d\n",cpu);
+		spin_lock(&per_cpu(rt_runnable_lock, cpu));
+		trr = per_cpu(top_rt_runnable, cpu);
+		spin_unlock(&per_cpu(rt_runnable_lock, cpu));
+		count += sysfs_emit_at(buf, count, "LONG RT_RUNNABLE: %lld\n",
+				atomic64_read(&(trr.count)));
+
+		for (i = 0; i < RT_RUNNABLE_ARR_SIZE; i++) {
+			long_rt_runnable = trr.rt_runnable[i];
+			sorted_trr[i].comm = kstrdup(long_rt_runnable.comm, GFP_KERNEL);
+			sorted_trr[i].prev_comm = kstrdup(long_rt_runnable.prev_comm, GFP_KERNEL);
+			sorted_trr[i].latency = long_rt_runnable.latency;
+		}
+		sort(sorted_trr, RT_RUNNABLE_ARR_SIZE, sizeof(struct rt_runnable),
+						runnable_latency_cmp, NULL);
+		for (i = 0; i < RT_RUNNABLE_ARR_SIZE; i++) {
+			count += sysfs_emit_at(buf, count, "%s %llu %s\n",
+				(const char *)sorted_trr[i].comm,
+				sorted_trr[i].latency,
+				(const char *)sorted_trr[i].prev_comm);
+		}
+		count += sysfs_emit_at(buf, count, "\n");
+	}
+	return count;
+}
+
+static ssize_t runnable_stats_reset_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf,
+					  size_t count)
+{
+	int cpu;
+	struct top_rt_runnable *trr;
+
+	for_each_possible_cpu(cpu) {
+		spin_lock(&per_cpu(rt_runnable_lock, cpu));
+		trr = &per_cpu(top_rt_runnable, cpu);
+		atomic64_set(&(trr->count), 0);
+		memset(trr->rt_runnable, 0, sizeof(struct top_rt_runnable));
+		spin_unlock(&per_cpu(rt_runnable_lock, cpu));
+	}
+	return count;
+}
+
+static ssize_t runnable_stats_enable_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf,
+					  size_t count)
+{
+	register_trace_sched_wakeup(vh_sched_wakeup_pixel_mod, NULL);
+	register_trace_sched_switch(vh_sched_switch_pixel_mod, NULL);
+	return count;
+}
+
+static ssize_t runnable_stats_disable_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf,
+					  size_t count)
+{
+	unregister_trace_sched_wakeup(vh_sched_wakeup_pixel_mod, NULL);
+	unregister_trace_sched_switch(vh_sched_switch_pixel_mod, NULL);
+	return count;
+}
+
 static struct kobj_attribute resume_latency_metrics_attr = __ATTR(resume_latency_metrics,
 							  0664,
 							  resume_latency_metrics_show,
@@ -556,12 +796,29 @@ static struct kobj_attribute irq_display_warning_attr = __ATTR(display_warning,
 							  0664,
 							  irq_display_warning_show,
 							  irq_display_warning_store);
-
 static struct kobj_attribute irq_stats_reset_attr = __ATTR(
 							stats_reset,
 							0200,
 							NULL,
 							irq_stats_reset_store);
+static struct kobj_attribute long_runnable_metrics_attr = __ATTR(stats,
+							  0444,
+							  long_runnable_metrics_show,
+							  NULL);
+
+static struct kobj_attribute runnable_stats_reset_attr = __ATTR(
+							stats_reset,
+							0200,
+							NULL,
+							runnable_stats_reset_store);
+static struct kobj_attribute runnable_stats_enable_attr = __ATTR(enable,
+							  0200,
+							  NULL,
+							  runnable_stats_enable_store);
+static struct kobj_attribute runnable_stats_disable_attr = __ATTR(disable,
+							  0200,
+							  NULL,
+							  runnable_stats_disable_store);
 
 static struct attribute *irq_attrs[] = {
 	&long_irq_metrics_attr.attr,
@@ -591,13 +848,29 @@ static const struct attribute_group resume_latency_attr_group = {
 	.name = "resume_latency"
 };
 
+static struct attribute *runnable_attrs[] = {
+	&long_runnable_metrics_attr.attr,
+	&runnable_stats_reset_attr.attr,
+	&runnable_stats_enable_attr.attr,
+	&runnable_stats_disable_attr.attr,
+	NULL
+};
+
+static const struct attribute_group runnable_attr_group = {
+	.attrs = runnable_attrs,
+	.name = "runnable"
+};
+
+
 /*********************************************************************
  *                  		INITIALIZE DRIVER                        *
  *********************************************************************/
 
 int perf_metrics_init(struct kobject *metrics_kobj)
 {
+	int cpu;
 	int ret = 0;
+
 	if (!metrics_kobj) {
 		pr_err("metrics_kobj is not initialized\n");
 		return -EINVAL;
@@ -610,6 +883,11 @@ int perf_metrics_init(struct kobject *metrics_kobj)
 		pr_err("failed to create irq folder\n");
 		return -ENOMEM;
 	}
+	if (sysfs_create_group(metrics_kobj, &runnable_attr_group)) {
+		pr_err("failed to create runnable folder\n");
+		return -ENOMEM;
+	}
+
 	spin_lock_init(&resume_latency_stats.resume_latency_stat_lock);
 	resume_latency_stats.resume_latency_threshold = RESUME_LATENCY_DEFAULT_THRESHOLD;
 	ret = register_trace_android_vh_early_resume_begin(
@@ -624,6 +902,7 @@ int perf_metrics_init(struct kobject *metrics_kobj)
 		pr_err("Register resume end vendor hook fail %d\n", ret);
 		return ret;
 	}
+
 	long_irq_stat.long_softirq_threshold = 10000;
 	long_irq_stat.long_irq_threshold = 500;
 	long_irq_stat.irq_storm_threshold_us = 500;
@@ -647,6 +926,18 @@ int perf_metrics_init(struct kobject *metrics_kobj)
 		pr_err("Register irq exit hook fail %d\n", ret);
 		return ret;
 	}
+
+	for_each_possible_cpu(cpu) {
+		spin_lock_init(&per_cpu(rt_runnable_lock, cpu));
+	}
+	ret = register_trace_sched_wakeup(vh_sched_wakeup_pixel_mod, NULL);
+	if (ret)
+		return ret;
+
+	ret = register_trace_sched_switch(vh_sched_switch_pixel_mod, NULL);
+	if (ret)
+		return ret;
+
 	pr_info("perf_metrics driver initialized! :D\n");
 	return ret;
 }
diff --git a/drivers/soc/google/vh/kernel/metrics/perf_metrics.h b/drivers/soc/google/vh/kernel/metrics/perf_metrics.h
index 1174ad157..79902d5c9 100644
--- a/drivers/soc/google/vh/kernel/metrics/perf_metrics.h
+++ b/drivers/soc/google/vh/kernel/metrics/perf_metrics.h
@@ -20,6 +20,8 @@
 #define MAX_IRQ_NUM 2048
 #define IRQ_ARR_LIMIT 100
 
+#define RT_RUNNABLE_ARR_SIZE 5
+
 #define LATENCY_CNT_SMALL (RESUME_LATENCY_BOUND_SMALL / RESUME_LATENCY_STEP_SMALL)
 #define LATENCY_CNT_MID ((RESUME_LATENCY_BOUND_MID - RESUME_LATENCY_BOUND_SMALL) / \
 	RESUME_LATENCY_STEP_MID)
@@ -27,35 +29,3 @@
 	RESUME_LATENCY_STEP_LARGE)
 #define RESUME_LATENCY_ARR_SIZE (LATENCY_CNT_SMALL + LATENCY_CNT_MID + LATENCY_CNT_LARGE + 1)
 
-struct irq_storm_data {
-	atomic64_t storm_count;
-	s64 max_storm_count;
-	atomic64_t irq_storm_start;
-};
-
-struct  resume_latency {
-	u64 resume_start;
-	u64 resume_end;
-	spinlock_t resume_latency_stat_lock;
-	s64 resume_count[RESUME_LATENCY_ARR_SIZE];
-	u64 resume_latency_max_ms;
-	u64 resume_latency_sum_ms;
-	u64 resume_latency_threshold;
-	bool display_warning;
-};
-
-struct long_irq {
-	ktime_t softirq_start[CONFIG_VH_SCHED_CPU_NR][NR_SOFTIRQS];
-	ktime_t irq_start[CONFIG_VH_SCHED_CPU_NR][MAX_IRQ_NUM];
-	atomic64_t long_softirq_count;
-	atomic64_t long_irq_count;
-	atomic64_t long_softirq_count_arr[CONFIG_VH_SCHED_CPU_NR];
-	atomic64_t long_irq_count_arr[CONFIG_VH_SCHED_CPU_NR];
-	s64 long_softirq_arr[NR_SOFTIRQS];
-	s64 long_irq_arr[MAX_IRQ_NUM];
-	struct irq_storm_data irq_storms[MAX_IRQ_NUM];
-	s64 long_softirq_threshold;
-	s64 long_irq_threshold;
-	s64 irq_storm_threshold_us;
-	bool display_warning;
-};
diff --git a/drivers/soc/google/vh/kernel/sched/sched_priv.h b/drivers/soc/google/vh/kernel/sched/sched_priv.h
index c1fdfca07..12487fb3c 100644
--- a/drivers/soc/google/vh/kernel/sched/sched_priv.h
+++ b/drivers/soc/google/vh/kernel/sched/sched_priv.h
@@ -456,6 +456,7 @@ static inline void init_vendor_task_struct(struct vendor_task_struct *v_tsk)
 	v_tsk->binder_task.active = false;
 	v_tsk->uclamp_pi[UCLAMP_MIN] = uclamp_none(UCLAMP_MIN);
 	v_tsk->uclamp_pi[UCLAMP_MAX] = uclamp_none(UCLAMP_MAX);
+	v_tsk->runnable_start_ns = -1;
 }
 
 int acpu_init(void);