diff options
author | Aurora zuma automerger <aurora-zuma-automerger@google.com> | 2023-03-30 15:16:46 +0000 |
---|---|---|
committer | Copybara-Service <copybara-worker@google.com> | 2023-04-23 23:29:11 -0700 |
commit | 7edc47df27e29a4b061fb4063f93ffb7ede85442 (patch) | |
tree | 556cb2ef15b05b5599a879a14048432b88f2adb8 | |
parent | d055791577688a8357ebecbff9abc74fe0426470 (diff) | |
download | zuma-7edc47df27e29a4b061fb4063f93ffb7ede85442.tar.gz |
gxp: [Copybara Auto Merge] Merge branch 'zuma' into 'android14-gs-pixel-5.15'
gcip: introduce gcip_usage_stats_max_watermark
Bug: 276474775
gcip: introduce gcip_usage_stats_thread_stats
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_counter
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_component_utilization
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_core_usage
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_dvfs_frequency_info
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_ops
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_{header,metric}
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_attr
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_metric_type
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats
Bug: 276474775 (repeat)
GCIP_HEADERS_REV_ID: 9aa973e0b2b1f4ec628bb181b7a08fed85d8f7bc
gcip: Conditionally include dma-iommu.h
gcip: add a new line to dvfs_freqs show
gcip: introduce gcip_usage_stats_max_watermark
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_thread_stats
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_counter
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_component_utilization
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_core_usage
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_dvfs_frequency_info
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_ops
Bug: 276474775 (repeat)
gcip: implement parsing metrics
Bug: 276474775 (repeat)
gcip: registers device attributes with default callbacks
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats_attr
Bug: 276474775 (repeat)
gcip: introduce gcip_usage_stats
Bug: 276474775 (repeat)
GCIP_MAIN_REV_ID: a7bf1859424f476d606b14abdc33ceb5a4d5ab7d
gxp: release client->sema before generating dump
Bug: 277863755
gxp: move release/unlink vmbox funcs to gxp-vd
Bug: 277863755 (repeat)
gxp: allow GXP_MAILBOX_UCI_RESPONSE ioctl from invalidated VD
Bug: 277863755 (repeat)
gxp: gxp_vd_invalidate will not flush UCI resps
Bug: 277863755 (repeat)
gxp: Move virt_core under HAS_COREDUMP
GitOrigin-RevId: f6194f10281863ee39893f30809c0f2948e34709
Change-Id: I25c920efb62c76b0366cfbece2ff226b11bdc858
-rw-r--r-- | gcip-kernel-driver/drivers/gcip/Makefile | 3 | ||||
-rw-r--r-- | gcip-kernel-driver/drivers/gcip/gcip-iommu.c | 5 | ||||
-rw-r--r-- | gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c | 1077 | ||||
-rw-r--r-- | gcip-kernel-driver/include/gcip/gcip-usage-stats.h | 634 | ||||
-rw-r--r-- | gxp-debug-dump.c | 2 | ||||
-rw-r--r-- | gxp-mcu-fs.c | 4 | ||||
-rw-r--r-- | gxp-mcu-platform.c | 70 | ||||
-rw-r--r-- | gxp-vd.c | 79 | ||||
-rw-r--r-- | gxp-vd.h | 31 |
9 files changed, 1830 insertions, 75 deletions
diff --git a/gcip-kernel-driver/drivers/gcip/Makefile b/gcip-kernel-driver/drivers/gcip/Makefile index 7af6c7e..55b4353 100644 --- a/gcip-kernel-driver/drivers/gcip/Makefile +++ b/gcip-kernel-driver/drivers/gcip/Makefile @@ -17,7 +17,8 @@ gcip-objs := gcip-alloc-helper.o \ gcip-mem-pool.o \ gcip-pm.o \ gcip-telemetry.o \ - gcip-thermal.o + gcip-thermal.o \ + gcip-usage-stats.o CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/gcip-kernel-driver/drivers/gcip/gcip-iommu.c b/gcip-kernel-driver/drivers/gcip/gcip-iommu.c index ab0ef51..8f6570f 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-iommu.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-iommu.c @@ -8,7 +8,6 @@ #include <linux/bitops.h> #include <linux/device.h> #include <linux/dma-direction.h> -#include <linux/dma-iommu.h> #include <linux/dma-mapping.h> #include <linux/genalloc.h> #include <linux/iova.h> @@ -30,6 +29,10 @@ #define HAS_IOVAD_BEST_FIT_ALGO (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) && \ (IS_ENABLED(CONFIG_GCIP_TEST) || IS_ENABLED(CONFIG_ANDROID))) +#if HAS_IOVAD_BEST_FIT_ALGO +#include <linux/dma-iommu.h> +#endif + /* Macros for manipulating @gcip_map_flags parameter. */ #define GCIP_MAP_FLAGS_GET_VALUE(ATTR, flags) \ (((flags) >> (GCIP_MAP_FLAGS_##ATTR##_OFFSET)) & \ diff --git a/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c b/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c new file mode 100644 index 0000000..876733e --- /dev/null +++ b/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c @@ -0,0 +1,1077 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Interface of managing the usage stats of IPs. + * + * Copyright (C) 2023 Google LLC + */ + +#include <linux/device.h> +#include <linux/hashtable.h> +#include <linux/kernel.h> +#include <linux/lockdep.h> +#include <linux/mutex.h> +#include <linux/string.h> +#include <linux/types.h> + +#include <gcip/gcip-usage-stats.h> + +typedef ssize_t (*show_t)(struct device *dev, struct device_attribute *dev_attr, char *buf); +typedef ssize_t (*store_t)(struct device *dev, struct device_attribute *dev_attr, const char *buf, + size_t count); + +/* + * Show callback which simply redirects to the user defined one. + * If GCIP doesn't have its own implementation because we expect that it won't be used in the real + * cases or the user has their own implementation for the specific metric, this callback will + * be used for the show function of the device attribute. + */ +static ssize_t gcip_usage_stats_user_defined_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + ssize_t ret = 0; + + if (attr->show) + ret = attr->show(dev, attr, buf, attr->ustats->data); + + return ret; +} + +/* + * Store callback which simply redirects to the user defined one. + * If GCIP doesn't have its own implementation because we expect that it won't be used in the real + * cases or the user has their own implementation for the specific metric, this callback will + * be used for the store function of the device attribute. + */ +static ssize_t gcip_usage_stats_user_defined_store(struct device *dev, + struct device_attribute *dev_attr, + const char *buf, size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + ssize_t ret = 0; + + if (attr->store) + ret = attr->store(dev, attr, buf, count, attr->ustats->data); + + return ret; +} + +/* Following functions are related to `CORE_USAGE` metrics. */ + +/* + * Returns the core usage entry of @uid and @core_id from @ustats->core_usage_htable hash table. + * Caller must hold @ustats->usage_stats_lock. + */ +static struct gcip_usage_stats_core_usage_uid_entry * +gcip_usage_stats_find_core_usage_entry_locked(int32_t uid, uint8_t core_id, + struct gcip_usage_stats *ustats) +{ + struct gcip_usage_stats_core_usage_uid_entry *uid_entry; + + lockdep_assert_held(&ustats->usage_stats_lock); + + hash_for_each_possible (ustats->core_usage_htable[core_id], uid_entry, node, uid) { + if (uid_entry->uid == uid) + return uid_entry; + } + + return NULL; +} + +/* Returns the 0-based index of @dvfs_freq from the frequency table. */ +static unsigned int gcip_usage_stats_find_dvfs_freq_index(struct gcip_usage_stats *ustats, + uint32_t dvfs_freq) +{ + int i, nums, idx = 0; + + mutex_lock(&ustats->dvfs_freqs_lock); + + /* + * Uses the frequency table, @ustats->dvfs_freqs, if the firmware has ever reported + * frequencies via `DVFS_FREQUENCY_INFO` metrics. + */ + if (ustats->dvfs_freqs_num) { + for (i = ustats->dvfs_freqs_num - 1; i >= 0; i--) { + if (dvfs_freq == ustats->dvfs_freqs[i]) + idx = i; + } + + if (i < 0) + dev_warn(ustats->dev, + "Failed to find the freq among the ones sent from the FW, freq=%u", + dvfs_freq); + + mutex_unlock(&ustats->dvfs_freqs_lock); + return idx; + } + + mutex_unlock(&ustats->dvfs_freqs_lock); + + /* Uses default one in case of the firmware has never reported supported frequencies. */ + nums = ustats->ops->get_default_dvfs_freqs_num(ustats->data); + if (nums <= 0) { + dev_warn_once(ustats->dev, "The kernel driver doesn't have default DVFS freqs"); + return 0; + } + + for (i = nums - 1; i >= 0; i--) { + if (dvfs_freq >= ustats->ops->get_default_dvfs_freq(i, ustats->data)) + return i; + } + + dev_warn(ustats->dev, + "Failed to find the freq from the default ones of the kernel driver, freq=%u", + dvfs_freq); + + return 0; +} + +/* + * Updates the entry of @uid in the core usage hash table of @core_id. + * If there is no entry for @uid, it will create one and insert it into the table. + * + * Called when the FW sent `CORE_USAGE` metrics. + */ +static void gcip_usage_stats_update_core_usage(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_core_usage *new, + int fw_metric_version) +{ + struct gcip_usage_stats_core_usage_uid_entry *uid_entry; + unsigned int state = gcip_usage_stats_find_dvfs_freq_index(ustats, new->operating_point); + uint8_t core_id = 0; + + if (fw_metric_version >= GCIP_USAGE_STATS_V2) + core_id = new->core_id; + + if (core_id >= ustats->subcomponents) { + dev_warn_once(ustats->dev, + "FW sent an invalid core_id for the core usage update, core_id=%u", + core_id); + return; + } + + mutex_lock(&ustats->usage_stats_lock); + + /* Finds the uid from @ustats->core_usage_htable first. */ + uid_entry = gcip_usage_stats_find_core_usage_entry_locked(new->uid, core_id, ustats); + if (uid_entry) { + uid_entry->time_in_state[state] += new->control_core_duration; + mutex_unlock(&ustats->usage_stats_lock); + return; + } + + dev_dbg(ustats->dev, "FW sent a new uid for the core usage update, uid=%d, core_id=%u", + new->uid, core_id); + + /* Allocates an entry for this uid. */ + uid_entry = devm_kzalloc(ustats->dev, sizeof(*uid_entry), GFP_KERNEL); + if (!uid_entry) { + dev_err(ustats->dev, + "Failed to allocate an entry of core usage hash table, uid=%d, core_id=%u", + new->uid, core_id); + mutex_unlock(&ustats->usage_stats_lock); + return; + } + + uid_entry->uid = new->uid; + uid_entry->time_in_state[state] += new->control_core_duration; + + /* Adds @uid_entry to the @ustats->core_usage_htable. */ + hash_add(ustats->core_usage_htable[core_id], &uid_entry->node, new->uid); + + mutex_unlock(&ustats->usage_stats_lock); +} + +/* Releases all entries in the core usage hash table of @core_id. */ +static void gcip_usage_stats_free_core_usage_core_entries_locked(struct gcip_usage_stats *ustats, + uint8_t core_id) +{ + unsigned int bkt; + struct gcip_usage_stats_core_usage_uid_entry *uid_entry; + struct hlist_node *tmp; + + lockdep_assert_held(&ustats->usage_stats_lock); + + hash_for_each_safe (ustats->core_usage_htable[core_id], bkt, tmp, uid_entry, node) { + hash_del(&uid_entry->node); + devm_kfree(ustats->dev, uid_entry); + } +} + +/* Releases all entries of all core usage hash tables. */ +static void gcip_usage_stats_free_core_usage_all_entries(struct gcip_usage_stats *ustats) +{ + int i; + + mutex_lock(&ustats->usage_stats_lock); + + for (i = 0; i < ustats->subcomponents; i++) + gcip_usage_stats_free_core_usage_core_entries_locked(ustats, i); + + mutex_unlock(&ustats->usage_stats_lock); +} + +/* + * Prints the core usage per uid in multiple arrays with the whitespace separation: + * <uid_0> <core_usage_0_1> <core_usage_0_2> ... + * <uid_1> <core_usage_1_1> <core_usage_1_2> ... + * ... + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_core_usage_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + struct gcip_usage_stats_core_usage_uid_entry *uid_entry; + int i, dvfs_freqs_num; + unsigned int bkt; + ssize_t written = 0; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->dvfs_freqs_lock); + + if (!ustats->dvfs_freqs_num) + dvfs_freqs_num = ustats->ops->get_default_dvfs_freqs_num(ustats->data); + else + dvfs_freqs_num = ustats->dvfs_freqs_num; + + mutex_unlock(&ustats->dvfs_freqs_lock); + mutex_lock(&ustats->usage_stats_lock); + + hash_for_each (ustats->core_usage_htable[attr->subcomponent], bkt, uid_entry, node) { + written += scnprintf(buf + written, PAGE_SIZE - written, "%d", uid_entry->uid); + + for (i = 0; i < dvfs_freqs_num; i++) + written += scnprintf(buf + written, PAGE_SIZE - written, " %lld", + uid_entry->time_in_state[i]); + + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); + } + + mutex_unlock(&ustats->usage_stats_lock); + + return written; +} + +/* + * Releases all the entries of the core usage hash table of the given core id, @attr->subcomponent. + * + * Called when the runtime writes the device attribute. + */ +static ssize_t gcip_usage_stats_core_usage_store(struct device *dev, + struct device_attribute *dev_attr, const char *buf, + size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + + mutex_lock(&ustats->usage_stats_lock); + gcip_usage_stats_free_core_usage_core_entries_locked(ustats, attr->subcomponent); + mutex_unlock(&ustats->usage_stats_lock); + + return count; +} + +/* Following functions are related to `COMPONENT_UTILIZATION` metrics. */ + +/* + * Updates the utilization of components. + * The value of utilization must be [0, 100]. + * + * Called when the FW sent `COMPONENT_UTILIZATION` metrics. + */ +static void +gcip_usage_stats_update_component_utilization(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_component_utilization *new, + uint16_t fw_metric_version) +{ + if (new->component < 0 || + new->component >= GCIP_USAGE_STATS_COMPONENT_UTILIZATION_NUM_TYPES) { + dev_warn_once(ustats->dev, "FW sent an invalid component utilization type, type=%d", + new->component); + return; + } + + if (new->utilization < 0 || new->utilization > 100) { + dev_warn_once(ustats->dev, + "FW sent an invalid component utilization value, type=%d, value=%d", + new->component, new->utilization); + return; + } + + mutex_lock(&ustats->usage_stats_lock); + ustats->component_utilization[new->component] = new->utilization; + mutex_unlock(&ustats->usage_stats_lock); +} + +/* + * Prints the utilization of the specific component. + * Note that this function also resets the utilization to 0. Therefore, we don't have a specific + * store function implementation for this stats. The `gcip_usage_stats_alloc_attrs` function will + * register the `gcip_usage_stats_user_defined_store` function as the store function if the kernel + * driver enables the write permission. + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_component_utilization_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + int32_t val; + ssize_t written; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->usage_stats_lock); + + val = ustats->component_utilization[attr->type]; + ustats->component_utilization[attr->type] = 0; + + mutex_unlock(&ustats->usage_stats_lock); + + written = scnprintf(buf, PAGE_SIZE, "%d\n", val); + if (written < 0) + return written; + + return written; +} + +/* Following functions are related to `COUNTER` metrics. */ + +/* + * Updates the counter which represents monotonically increased occurrences such as workloads + * and preemptions. + * + * Called when the FW sent `COUNTER` metrics. + */ +static void gcip_usage_stats_update_counter(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_counter *new, + uint16_t fw_metric_version) +{ + uint8_t component_id = 0; + + if (new->type < 0 || new->type >= GCIP_USAGE_STATS_COUNTER_NUM_TYPES) { + dev_warn_once(ustats->dev, "FW sent an invalid counter type, type=%d", new->type); + return; + } + + if (fw_metric_version >= GCIP_USAGE_STATS_V2) + component_id = new->component_id; + + if (component_id >= ustats->subcomponents) { + dev_warn_once( + ustats->dev, + "FW sent an invalid component_id for the counter update, component_id=%d", + component_id); + return; + } + + mutex_lock(&ustats->usage_stats_lock); + ustats->counter[component_id][new->type] += new->value; + mutex_unlock(&ustats->usage_stats_lock); +} + +/* + * Prints the value(s) of counter. + * If the @attr->subcomponent is `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS`, it will print the + * counters of all subcomponents in one array with whitespace separation. Otherwise, it will print + * the counter of the specific subcomponent. + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_counter_show(struct device *dev, struct device_attribute *dev_attr, + char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + ssize_t written = 0; + int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + int i; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->usage_stats_lock); + + if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { + for (i = 0; i < ustats->subcomponents; i++) { + /* Prints a blank only when @i is bigger than 0. */ + written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%lld", i, " ", + ustats->counter[i][attr->type]); + } + } else { + written += scnprintf(buf + written, PAGE_SIZE - written, "%lld", + ustats->counter[subcomponent][attr->type]); + } + + mutex_unlock(&ustats->usage_stats_lock); + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); + + return written; +} + +/* + * Clears the value(s) of counter. + * As described in the show function, it will clears the counters of all subcomponents when + * @attr->subcomponent is `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS`. Otherwise, it will clear the + * counter of the specific subcomponent. + * + * Called when the runtime writes the device attribute. + */ +static ssize_t gcip_usage_stats_counter_store(struct device *dev, struct device_attribute *dev_attr, + const char *buf, size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + int i; + + mutex_lock(&ustats->usage_stats_lock); + + if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { + for (i = 0; i < ustats->subcomponents; i++) + ustats->counter[i][attr->type] = 0; + } else { + ustats->counter[subcomponent][attr->type] = 0; + } + + mutex_unlock(&ustats->usage_stats_lock); + + return count; +} + +/* Following functions are related to `THREAD_STATISTICS` metrics. */ + +/* + * Updates the max stack usage of the @new->thread_id type of thread. + * + * Called when the FW sent `THREAD_STATISTICS` metrics. + */ +static void gcip_usage_stats_update_thread_stats(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_thread_stats *new, + uint16_t fw_metric_version) +{ + if (new->thread_id < 0 || new->thread_id >= GCIP_USAGE_STATS_THREAD_NUM_TYPES) { + dev_warn_once(ustats->dev, "FW sent an invalid thread_id, thread_id=%d", + new->thread_id); + return; + } + + mutex_lock(&ustats->usage_stats_lock); + + if (new->max_stack_usage_bytes > ustats->thread_max_stack_usage[new->thread_id]) + ustats->thread_max_stack_usage[new->thread_id] = new->max_stack_usage_bytes; + + mutex_unlock(&ustats->usage_stats_lock); +} + +/* + * Prints the max stack usage of each thread with tab separation. + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_thread_stats_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + int i; + ssize_t written = 0; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->usage_stats_lock); + + for (i = 0; i < GCIP_USAGE_STATS_THREAD_NUM_TYPES; i++) { + if (!ustats->thread_max_stack_usage[i]) + continue; + written += scnprintf(buf + written, PAGE_SIZE - written, "%u\t%u\n", i, + ustats->thread_max_stack_usage[i]); + } + + mutex_unlock(&ustats->usage_stats_lock); + + return written; +} + +/* + * Clears the max usage of all threads to 0. + * + * Called when the runtime writes the device attribute. + */ +static ssize_t gcip_usage_stats_thread_stats_store(struct device *dev, + struct device_attribute *dev_attr, + const char *buf, size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + + mutex_lock(&ustats->usage_stats_lock); + memset(ustats->thread_max_stack_usage, 0, sizeof(ustats->thread_max_stack_usage)); + mutex_unlock(&ustats->usage_stats_lock); + return count; +} + +/* Following functions are related to `MAX_WATERMARK` metrics. */ + +/* + * Updates the @new->type type of max watermark of @new->component_id of component. + * + * Called when the FW sent `MAX_WATERMARK` metrics. + */ +static void gcip_usage_stats_update_max_watermark(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_max_watermark *new, + uint16_t fw_metric_version) +{ + uint8_t component_id = 0; + + if (new->type < 0 || new->type >= GCIP_USAGE_STATS_MAX_WATERMARK_NUM_TYPES) { + dev_warn_once(ustats->dev, "FW sent an invalid max watermark type, type=%d", + new->type); + return; + } + + if (fw_metric_version >= GCIP_USAGE_STATS_V2) + component_id = new->component_id; + + if (component_id >= ustats->subcomponents) { + dev_warn_once( + ustats->dev, + "FW sent an invalid component_id for the max watermark update, component_id=%d", + component_id); + return; + } + + mutex_lock(&ustats->usage_stats_lock); + + if (new->value > ustats->max_watermark[component_id][new->type]) + ustats->max_watermark[component_id][new->type] = new->value; + + mutex_unlock(&ustats->usage_stats_lock); +} + +/* + * Printe the value(s) of max watermark. + * + * If the @attr->subcomponent is `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS`, it will print the + * values of all subcomponents in one array with whitespace separation. Otherwise, it will print + * the value of the specific subcomponent. + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_max_watermark_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + ssize_t written = 0; + int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + int i; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->usage_stats_lock); + + if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { + for (i = 0; i < ustats->subcomponents; i++) { + /* Prints a blank only when @i is bigger than 0. */ + written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%lld", i, " ", + ustats->max_watermark[i][attr->type]); + } + } else { + written += scnprintf(buf + written, PAGE_SIZE - written, "%lld", + ustats->max_watermark[subcomponent][attr->type]); + } + + mutex_unlock(&ustats->usage_stats_lock); + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); + + return written; +} + +/* + * Clears the value(s) of max watermark. + * As described in the show function, it will clears the values of all subcomponents when + * @attr->subcomponent is `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS`. Otherwise, it will clear the + * value of the specific subcomponent. + * + * Called when the runtime writes the device attribute. + */ +static ssize_t gcip_usage_stats_max_watermark_store(struct device *dev, + struct device_attribute *dev_attr, + const char *buf, size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + int i; + + mutex_lock(&ustats->usage_stats_lock); + + if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { + for (i = 0; i < ustats->subcomponents; i++) + ustats->max_watermark[i][attr->type] = 0; + } else { + ustats->max_watermark[subcomponent][attr->type] = 0; + } + + mutex_unlock(&ustats->usage_stats_lock); + + return count; +} + +/* Following functions are related to `DVFS_FREQUENCY_INFO` metrics. */ + +/* + * Updates the DVFS freq table with the ones sent by the FW. + * + * Until the runtime flushes them by writing the device attribute (i.e., the + * `gcip_usage_stats_dvfs_freqs_store` function is called), they will be used instead of the + * default freqs of the kernel driver. + * + * Called when the FW sent `DVFS_FREQUENCY_INFO` metrics. + */ +static void gcip_usage_stats_update_dvfs_freq_info(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_dvfs_frequency_info *new, + uint16_t fw_metric_version) +{ + int i; + + mutex_lock(&ustats->dvfs_freqs_lock); + + for (i = 0; i < ustats->dvfs_freqs_num; i++) + if (ustats->dvfs_freqs[i] == new->supported_frequency) + goto out; + + if (ustats->dvfs_freqs_num >= GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM) { + dev_warn(ustats->dev, "FW sent more DVFS freqs than the kernel driver can handle"); + goto out; + } + + ustats->dvfs_freqs[ustats->dvfs_freqs_num++] = new->supported_frequency; +out: + mutex_unlock(&ustats->dvfs_freqs_lock); +} + +/* Flushes the DVFS freqs from the FW by simply setting the number of freqs in the table to 0. */ +static void gcip_usage_stats_reset_dvfs_freqs(struct gcip_usage_stats *ustats) +{ + mutex_lock(&ustats->dvfs_freqs_lock); + ustats->dvfs_freqs_num = 0; + mutex_unlock(&ustats->dvfs_freqs_lock); +} + +/* + * Prints the list of available DVFS freqs in an array with the whitespace separation. + * + * If the FW has sent `DVFS_FREQUENCY_INFO` metrics, they will be printed. Otherwise, the default + * ones from the kernel driver will be printed. + * + * Called when the runtime reads the device attribute. + */ +static ssize_t gcip_usage_stats_dvfs_freqs_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + struct gcip_usage_stats *ustats = attr->ustats; + int i, dvfs_freqs_num; + ssize_t written = 0; + + ustats->ops->update_usage_kci(ustats->data); + + mutex_lock(&ustats->dvfs_freqs_lock); + + if (!ustats->dvfs_freqs_num) { + mutex_unlock(&ustats->dvfs_freqs_lock); + dvfs_freqs_num = ustats->ops->get_default_dvfs_freqs_num(ustats->data); + for (i = 0; i < dvfs_freqs_num; i++) + written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%d", i, " ", + ustats->ops->get_default_dvfs_freq(i, ustats->data)); + } else { + dvfs_freqs_num = ustats->dvfs_freqs_num; + for (i = 0; i < dvfs_freqs_num; i++) + written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%u", i, " ", + ustats->dvfs_freqs[i]); + mutex_unlock(&ustats->dvfs_freqs_lock); + } + + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); + + return written; +} + +/* + * Flushes the DVFS freqs sent by the FW. + * + * Called when the runtime writes the device attribute. + */ +static ssize_t gcip_usage_stats_dvfs_freqs_store(struct device *dev, + struct device_attribute *dev_attr, const char *buf, + size_t count) +{ + struct gcip_usage_stats_attr *attr = + container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); + + gcip_usage_stats_reset_dvfs_freqs(attr->ustats); + + return count; +} + +/* Parses header part of @buf. */ +static void *gcip_usage_stats_parse_header(struct gcip_usage_stats *ustats, void *buf, + uint32_t *num_metrics, uint32_t *metric_size, + uint16_t *fw_metric_version) +{ + struct gcip_usage_stats_header *header = buf; + struct gcip_usage_stats_header_v1 *header_v1 = buf; + + if (ustats->version <= GCIP_USAGE_STATS_V1) { + *num_metrics = header_v1->num_metrics; + *metric_size = header_v1->metric_size; + *fw_metric_version = GCIP_USAGE_STATS_V1; + buf += sizeof(*header_v1); + } else { + *num_metrics = header->num_metrics; + *metric_size = header->metric_size; + *fw_metric_version = header->version; + buf += sizeof(*header); + } + + return buf; +} + +/* + * Fills out required information of device attribute such as show and store callbacks. Also, + * checks the validity of it. + */ +static int gcip_usage_stats_fill_attr(struct gcip_usage_stats *ustats, + struct gcip_usage_stats_attr *attr, const show_t show, + const store_t store) +{ + struct device_attribute *dev_attr = &attr->dev_attr; + + /* + * CORE_USAGE metric doesn't support showing stats for all subcomponents in one device + * attribute. Because its printing format is complicated. + */ + if (attr->metric == GCIP_USAGE_STATS_METRIC_TYPE_CORE_USAGE && + attr->subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) + return -EINVAL; + + /* + * For metrics which store stats per subcomponent, we have to check whether the caller set + * a proper subcomponent index. + */ + if (attr->subcomponent != GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS && + (attr->subcomponent < 0 || attr->subcomponent >= ustats->subcomponents)) + return -EINVAL; + + switch (attr->metric) { + case GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION: + if (attr->type >= GCIP_USAGE_STATS_COMPONENT_UTILIZATION_NUM_TYPES) + return -EINVAL; + break; + case GCIP_USAGE_STATS_METRIC_TYPE_COUNTER: + if (attr->type >= GCIP_USAGE_STATS_COUNTER_NUM_TYPES) + return -EINVAL; + break; + case GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK: + if (attr->type >= GCIP_USAGE_STATS_MAX_WATERMARK_NUM_TYPES) + return -EINVAL; + break; + default: + break; + } + + /* + * If the user defines its own show/store callbacks (@attr->show or @attr->store), use the + * functions which simply redirect to the user-defined callbacks. + */ + if (attr->mode == GCIP_USAGE_STATS_MODE_RW || attr->mode == GCIP_USAGE_STATS_MODE_RO) + dev_attr->show = attr->show ? gcip_usage_stats_user_defined_show : show; + if (attr->mode == GCIP_USAGE_STATS_MODE_RW || attr->mode == GCIP_USAGE_STATS_MODE_WO) + dev_attr->store = attr->store ? gcip_usage_stats_user_defined_store : store; + + attr->ustats = ustats; + dev_attr->attr.mode = attr->mode; + dev_attr->attr.name = attr->name; + + return 0; +} + +/* + * Allocates @ustats->attrs, a pointer array of `struct attribute`. Each pointers will indicate the + * `struct attribute` instances of the @args->attrs[]->dev_attr.attr. + * It will fill out show and store callbacks of each device attribute and the allocated + * @ustats->attr will be set to the @ustats->group.attr to register them. + */ +static int gcip_usage_stats_alloc_attrs(struct gcip_usage_stats *ustats, + const struct gcip_usage_stats_args *args) +{ + int i, ret; + struct gcip_usage_stats_attr *attr; + + ustats->attrs = + devm_kcalloc(ustats->dev, args->num_attrs + 1, sizeof(*ustats->attrs), GFP_KERNEL); + if (!ustats->attrs) + return -ENOMEM; + + /* TODO: fill @ustats->attrs according to the metrics. */ + for (i = 0; i < args->num_attrs; i++) { + attr = args->attrs[i]; + + switch (attr->metric) { + case GCIP_USAGE_STATS_METRIC_TYPE_CORE_USAGE: + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_core_usage_show, + gcip_usage_stats_core_usage_store); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION: + ret = gcip_usage_stats_fill_attr( + ustats, attr, gcip_usage_stats_component_utilization_show, + gcip_usage_stats_user_defined_store); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_COUNTER: + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_counter_show, + gcip_usage_stats_counter_store); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_THREAD_STATS: + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_thread_stats_show, + gcip_usage_stats_thread_stats_store); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK: + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_max_watermark_show, + gcip_usage_stats_max_watermark_store); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_DVFS_FREQUENCY_INFO: + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_dvfs_freqs_show, + gcip_usage_stats_dvfs_freqs_store); + break; + default: + dev_warn( + ustats->dev, + "Invalid usage stats metric, use user defined callbacks (metric=%d)", + attr->metric); + ret = gcip_usage_stats_fill_attr(ustats, attr, + gcip_usage_stats_user_defined_show, + gcip_usage_stats_user_defined_store); + break; + } + + if (ret) { + devm_kfree(ustats->dev, ustats->attrs); + return ret; + } + + ustats->attrs[i] = &attr->dev_attr.attr; + } + + ustats->attrs[args->num_attrs] = NULL; + ustats->group.attrs = ustats->attrs; + + return 0; +} + +/* Releases @ustats->attrs allocated by the `gcip_usage_stats_alloc_attrs` function. */ +static void gcip_usage_stats_free_attrs(struct gcip_usage_stats *ustats) +{ + devm_kfree(ustats->dev, ustats->attrs); +} + +/* Allocates arrays which store the statistics per subcomponent. */ +static int gcip_usage_stats_alloc_stats(struct gcip_usage_stats *ustats) +{ + int i; + + ustats->core_usage_htable = devm_kcalloc(ustats->dev, ustats->subcomponents, + sizeof(*ustats->core_usage_htable), GFP_KERNEL); + if (!ustats->core_usage_htable) + return -ENOMEM; + + ustats->counter = devm_kcalloc(ustats->dev, ustats->subcomponents, sizeof(*ustats->counter), + GFP_KERNEL); + if (!ustats->counter) + goto err_free_core_usage_htable; + + ustats->max_watermark = devm_kcalloc(ustats->dev, ustats->subcomponents, + sizeof(*ustats->max_watermark), GFP_KERNEL); + if (!ustats->max_watermark) + goto err_free_counter; + + for (i = 0; i < ustats->subcomponents; i++) + hash_init(ustats->core_usage_htable[i]); + + return 0; + +err_free_counter: + devm_kfree(ustats->dev, ustats->counter); +err_free_core_usage_htable: + devm_kfree(ustats->dev, ustats->core_usage_htable); + return -ENOMEM; +} + +/* Releases arrays which are allocated by the `gcip_usage_stats_alloc_stats` function. */ +static void gcip_usage_stats_free_stats(struct gcip_usage_stats *ustats) +{ + devm_kfree(ustats->dev, ustats->max_watermark); + devm_kfree(ustats->dev, ustats->counter); + devm_kfree(ustats->dev, ustats->core_usage_htable); +} + +/* Sets operators to the @ustats from @args. */ +static int gcip_usage_stats_set_ops(struct gcip_usage_stats *ustats, + const struct gcip_usage_stats_args *args) +{ + if (!args->ops->update_usage_kci || !args->ops->get_default_dvfs_freqs_num || + !args->ops->get_default_dvfs_freq) + return -EINVAL; + + ustats->ops = args->ops; + + return 0; +} + +int gcip_usage_stats_init(struct gcip_usage_stats *ustats, const struct gcip_usage_stats_args *args) +{ + int ret; + + if (args->version < GCIP_USAGE_STATS_V1 || args->version > GCIP_USAGE_STATS_V2) + return -EINVAL; + + if (!args->dev) + return -EINVAL; + + if (args->subcomponents < 1) + return -EINVAL; + + ustats->version = args->version; + ustats->subcomponents = args->subcomponents; + ustats->dev = args->dev; + ustats->data = args->data; + mutex_init(&ustats->usage_stats_lock); + mutex_init(&ustats->dvfs_freqs_lock); + ustats->dvfs_freqs_num = 0; + + ret = gcip_usage_stats_set_ops(ustats, args); + if (ret) + return ret; + + ret = gcip_usage_stats_alloc_stats(ustats); + if (ret) + return ret; + + ret = gcip_usage_stats_alloc_attrs(ustats, args); + if (ret) + goto err_free_stats; + + ret = device_add_group(ustats->dev, &ustats->group); + if (ret) + goto err_free_attrs; + + return 0; + +err_free_attrs: + gcip_usage_stats_free_attrs(ustats); +err_free_stats: + gcip_usage_stats_free_stats(ustats); + return ret; +} + +void gcip_usage_stats_exit(struct gcip_usage_stats *ustats) +{ + device_remove_group(ustats->dev, &ustats->group); + gcip_usage_stats_reset_dvfs_freqs(ustats); + gcip_usage_stats_free_core_usage_all_entries(ustats); + gcip_usage_stats_free_stats(ustats); + gcip_usage_stats_free_attrs(ustats); +} + +void gcip_usage_stats_process_buffer(struct gcip_usage_stats *ustats, void *buf) +{ + struct gcip_usage_stats_metric *metric; + uint32_t num_metrics; + uint32_t metric_size; + /* + * Stores the version of metrics that the firmware is using. + * If the version of the firmware and the kernel driver are mismatching, we have to parse + * @buf according to the lower version. + */ + uint16_t fw_metric_version; + int i; + + metric = gcip_usage_stats_parse_header(ustats, buf, &num_metrics, &metric_size, + &fw_metric_version); + + /* Firmware sent metrics which cannot be parsed. */ + if (fw_metric_version == GCIP_USAGE_STATS_V1 && + metric_size != GCIP_USAGE_STATS_METRIC_SIZE_V1) { + dev_err_once(ustats->dev, + "FW sent V1 metrics with invalid size (expected=%d, actual=%u)", + GCIP_USAGE_STATS_METRIC_SIZE_V1, metric_size); + return; + } + + /* The metric version of the firmware is higher than the kernel driver. */ + if (fw_metric_version >= GCIP_USAGE_STATS_VERSION_UPPER_BOUND || + metric_size > sizeof(struct gcip_usage_stats_metric)) + dev_warn_once( + ustats->dev, + "FW metrics are later version with unknown fields (expected=%zu, actual=%u, fw_metric_version=%u)", + sizeof(struct gcip_usage_stats_metric), metric_size, fw_metric_version); + + for (i = 0; i < num_metrics; i++) { + switch (metric->type) { + case GCIP_USAGE_STATS_METRIC_TYPE_CORE_USAGE: + gcip_usage_stats_update_core_usage(ustats, &metric->core_usage, + fw_metric_version); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION: + gcip_usage_stats_update_component_utilization( + ustats, &metric->component_utilization, fw_metric_version); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_COUNTER: + gcip_usage_stats_update_counter(ustats, &metric->counter, + fw_metric_version); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_THREAD_STATS: + gcip_usage_stats_update_thread_stats(ustats, &metric->thread_stats, + fw_metric_version); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK: + gcip_usage_stats_update_max_watermark(ustats, &metric->max_watermark, + fw_metric_version); + break; + case GCIP_USAGE_STATS_METRIC_TYPE_DVFS_FREQUENCY_INFO: + gcip_usage_stats_update_dvfs_freq_info(ustats, &metric->dvfs_frequency_info, + fw_metric_version); + break; + default: + dev_warn(ustats->dev, + "Invalid usage stats metric, skip parsing it (type=%d)", + metric->type); + break; + } + + metric = (struct gcip_usage_stats_metric *)((void *)metric + metric_size); + } +} diff --git a/gcip-kernel-driver/include/gcip/gcip-usage-stats.h b/gcip-kernel-driver/include/gcip/gcip-usage-stats.h new file mode 100644 index 0000000..a20fe33 --- /dev/null +++ b/gcip-kernel-driver/include/gcip/gcip-usage-stats.h @@ -0,0 +1,634 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Interface of managing the usage stats of IPs. + * + * Copyright (C) 2023 Google LLC + */ + +#ifndef __GCIP_USAGE_STATS_H__ +#define __GCIP_USAGE_STATS_H__ + +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/hashtable.h> +#include <linux/mutex.h> +#include <linux/stringify.h> +#include <linux/types.h> + +/* Attribute read/write mode (permissions). */ +#define GCIP_USAGE_STATS_MODE_RO 0444 +#define GCIP_USAGE_STATS_MODE_WO 0200 +#define GCIP_USAGE_STATS_MODE_RW (GCIP_USAGE_STATS_MODE_RO | GCIP_USAGE_STATS_MODE_WO) + +/* Macros which generate `struct gcip_usage_stats_attr` instances easily. */ +#define GCIP_USAGE_STATS_ATTR(_metric, _type, _subcomponent, _name, _mode, _show, _store) \ + struct gcip_usage_stats_attr gcip_usage_stats_attr_##_name = { \ + .metric = _metric, \ + .type = _type, \ + .subcomponent = _subcomponent, \ + .name = __stringify(_name), \ + .mode = _mode, \ + .show = _show, \ + .store = _store, \ + } + +#define GCIP_USAGE_STATS_ATTR_RW(metric, type, subcomponent, name, show, store) \ + GCIP_USAGE_STATS_ATTR(metric, type, subcomponent, name, GCIP_USAGE_STATS_MODE_RW, show, \ + store) + +#define GCIP_USAGE_STATS_ATTR_RO(metric, type, subcomponent, name, show) \ + GCIP_USAGE_STATS_ATTR(metric, type, subcomponent, name, GCIP_USAGE_STATS_MODE_RO, show, \ + NULL) + +#define GCIP_USAGE_STATS_ATTR_WO(metric, type, subcomponent, name, store) \ + GCIP_USAGE_STATS_ATTR(metric, type, subcomponent, name, GCIP_USAGE_STATS_MODE_WO, NULL, \ + store) + +/* + * Set a device attribute to show/store all subcomponents instead of one specific subcomponent. + * See @subcomponents field of `struct gcip_usage_stats_attr`. + */ +#define GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS -1 + +/* + * Size of the metric v1 in bytes. + * + * We decided to use 20 bytes for the V1 metrics. However, from V2, we increased the size of it to + * 24 bytes which is the same as `sizeof(struct gcip_usage_stats_metric)` to collect more stats + * information. To verify whether the firmware sent valid V1 size metrics, keep it as a macro. + */ +#define GCIP_USAGE_STATS_METRIC_SIZE_V1 20 + +/* Max number of frequencies to support. */ +#define GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM 10 + +struct gcip_usage_stats_attr; + +typedef ssize_t (*gcip_usage_stats_show_t)(struct device *dev, struct gcip_usage_stats_attr *attr, + char *buf, void *data); +typedef ssize_t (*gcip_usage_stats_store_t)(struct device *dev, struct gcip_usage_stats_attr *attr, + const char *buf, size_t count, void *data); + +/* + * The version of metrics. + * The format of header or the size of metrics would be different from the versions. However, + * the metric implementation will be shared. See each metric implementation to know which fields + * can be accessed according to the versions. + */ +enum gcip_usage_stats_version { + /* + * In V1, the headers must have the format of the `struct gcip_usage_stats_header_v1` and + * the size of metrics must be GCIP_USAGE_STATS_METRIC_SIZE_V1. + */ + GCIP_USAGE_STATS_V1 = 1, + /* + * In V2, the headers must have the format of the `struct gcip_usage_stats_header` and + * the size of metrics must be `sizeof(struct gcip_usage_stats_metric)`. + */ + GCIP_USAGE_STATS_V2 = 2, + /* Version of metrics must be lower than this. */ + GCIP_USAGE_STATS_VERSION_UPPER_BOUND, +}; + +/* Hash bits which will be used when initializing @ustats->core_usage_htable. */ +#define GCIP_USAGE_STATS_UID_HASH_BITS 3 + +/* Must be kept in sync with firmware `enum class UsageTrackerMetric::Type`. */ +enum gcip_usage_stats_metric_type { + GCIP_USAGE_STATS_METRIC_TYPE_RESERVED, + GCIP_USAGE_STATS_METRIC_TYPE_CORE_USAGE, + GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION, + GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_METRIC_TYPE_THREAD_STATS, + GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_METRIC_TYPE_DVFS_FREQUENCY_INFO, +}; + +/* + * Encapsulates core usage information of a specific application. + * Must be kept in sync with firmware `struct CoreUsage`. + */ +struct gcip_usage_stats_core_usage { + /* + * The applications global identifier. + * This value IS NOT the virtual identifier assigned by the accelerator kernel driver, + * this is the user ID of the application assigned by Linux kernel. + */ + int32_t uid; + /* The frequency (kHz) represented by this report. */ + uint32_t operating_point; + /* Utilization time in microseconds (us). */ + uint32_t control_core_duration; + + /* Following fields must not be accessed in lower than V2. */ + + /* + * The compute core is represented by this metric on DSP. + * The TPU does not personalize core usages by DVFS and can expect this value to be 0 at + * all times. For DSP the value will be either 0, 1, or 2. + */ + uint8_t core_id; + /* Reserved. */ + uint8_t reserved[3]; +} __packed; + +/* + * Hash table entry which stores core usage per uid. + * It will be added to the hash table of its subcomponent, @ustats->core_usage_htable[], using @uid + * as the key and itself as the value. + */ +struct gcip_usage_stats_core_usage_uid_entry { + int32_t uid; + uint64_t time_in_state[GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM]; + struct hlist_node node; +}; + +/* + * An enum to represent the different activity components we can track metrics for. + * Must be kept in sync with firmware `enum class Component`. + */ +enum gcip_usage_stats_component_utilization_type { + /* The entire IP Block. */ + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_IP, + /* A compute core. */ + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_CORES, + + /* The number of total types. Must be located at the end of this enum. */ + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_NUM_TYPES, +}; + +/* + * Encapsulates information about utilization of a component. + * Must be kept in sync with firmware `struct ComponentActivity`. + */ +struct gcip_usage_stats_component_utilization { + /* Type of component. */ + enum gcip_usage_stats_component_utilization_type component; + /* + * The percentage of time the component was active over the collection interval. + * This value is strictly between 0 and 100. + */ + int32_t utilization; + + /* Following fields must not be accessed in version 1. */ + + /* Reserved. */ + uint32_t reserved[2]; +} __packed; + +/* + * Defines different counter types we track. + * Must be kept in sync with firmware `enum class CounterType`. + */ +enum gcip_usage_stats_counter_type { + /* Active TPU cycles. */ + GCIP_USAGE_STATS_COUNTER_TPU_ACTIVIY_CYCLES, + /* The number of stalls caused by throttling. */ + GCIP_USAGE_STATS_COUNTER_TPU_THROTTLE_STALLS, + /* Number of TPU inferences / DSP workloads. */ + GCIP_USAGE_STATS_COUNTER_WORKLOAD, + /* Number of TPU offload op invocations. */ + GCIP_USAGE_STATS_COUNTER_TPU_OP, + /* Number of times a TPU op invocation used its cache parameters. */ + GCIP_USAGE_STATS_COUNTER_PARAM_CACHING_HIT, + /* Number of times a TPU op invocation had to cache its parameters. */ + GCIP_USAGE_STATS_COUNTER_PARAM_CACHING_MISS, + /* + * Number of times preemptions (either software or hardware) occurred between different + * clients. + * - Hardware preemption: The preemption which occurs between QoS classes. E.g., Realtime + * QoS class has a higher priority than Best-Effort QoS class. In + * this case, the current workload will be stopped at the next + * scalar fence, saved and will be restored after the new higher + * priority workload is completed. + * - Software preemption: The preemption which occurs between workloads with the same QoS + * class, but different priorities. The on-going workload is allowed + * to be completed before the higher priority one begins to execute. + * In the case of TPU, one TPU offload operation can be cut into + * multiple chunks (workloads) and it allows the higher priority + * offload to have the chance to preempt after the current chunk is + * processed. + */ + GCIP_USAGE_STATS_COUNTER_CONTEXT_PREEMPTIONS, + /* Number of times hardware preemptions occurred. */ + GCIP_USAGE_STATS_COUNTER_HW_PREEMPTIONS, + /* + * The total time in microseconds spent saving a hardware context during hardware + * preemption. + */ + GCIP_USAGE_STATS_COUNTER_TOTAL_HW_CONTEXT_SAVE_TIME, + /* The total time in microseconds spent waiting to hit a scalar fence. */ + GCIP_USAGE_STATS_COUNTER_TOTAL_SCALAR_FENCE_WAIT_TIME, + /* The number of times the Pipeline::Suspend function takes longer than SLA time. */ + GCIP_USAGE_STATS_COUNTER_NUM_OF_LONG_SUSPENDS, + /* The number of times a compute core experienced a context switch. */ + GCIP_USAGE_STATS_COUNTER_CONTEXT_SWITCHES, + /* The number of times a TPU cluster reconfiguration occurred. */ + GCIP_USAGE_STATS_COUNTER_NUM_OF_RECONFIGURATIONS, + /* + * The number of times a TPU cluster reconfiguration occurred and was strictly motivated by + * a preemption. + */ + GCIP_USAGE_STATS_COUNTER_NUM_OF_RECONFIGURATIONS_BY_PREEMPTION, + + /* The number of total types. Must be located at the end of this enum. */ + GCIP_USAGE_STATS_COUNTER_NUM_TYPES, +}; + +/* + * Generic counter. Only reported if it has a value larger than 0. + * Must be kept in sync with firmware `struct Counter`. + */ +struct gcip_usage_stats_counter { + /* What it counts. */ + enum gcip_usage_stats_counter_type type; + /* Accumulated value since last initialization. */ + uint64_t value; + + /* Following fields must not be accessed in version 1. */ + + /* An identifier that personalizes the represented hardware for counters. */ + uint8_t component_id; + /* Reserved. */ + uint8_t reserved[3]; +} __packed; + +/* + * An enum to identify the tracked firmware threads. + * Must be kept in sync with firmware `enum class UsageTrackerThreadId`. + */ +enum gcip_usage_stats_thread_stats_thread_id { + /* The entry thread for the firmware. */ + GCIP_USAGE_STATS_THREAD_MAIN_TASK, + /* + * The thread that processes commands from the kernel and sends commands reversely in some + * cases, e.g., firmware crashes. + */ + GCIP_USAGE_STATS_THREAD_KCI_HANDLER, + /* The thread that determines the commanded power state of the system. */ + GCIP_USAGE_STATS_THREAD_POWER_ADMINISTRATOR, + /* The thread responsible for coordinating and dispatching workloads. */ + GCIP_USAGE_STATS_THREAD_SCHEDULER, + /* The thread that handles VII commands from TPU clients. */ + GCIP_USAGE_STATS_THREAD_VII_HANDLER, + /* + * The multi-core coordination thread that shares complex assignments with other TPU cores. + */ + GCIP_USAGE_STATS_THREAD_MCP_GRAPH_DRIVER, + /* The single-core coordination thread that handles local-only graphs. */ + GCIP_USAGE_STATS_THREAD_SCP_GRAPH_DRIVER, + /* The thread that coordinates the progress of scalar and tile TPU workloads. */ + GCIP_USAGE_STATS_THREAD_TPU_DRIVER, + /* Orchestrates restarting client threads when there is a fatal error in the pipeline. */ + GCIP_USAGE_STATS_THREAD_RESTART_HANDLER, + /* Used for polling some state but not blocking other threads from execution. */ + GCIP_USAGE_STATS_THREAD_POLL_SERVICE, + /* Schedules DMAs on the main DMA engine. */ + GCIP_USAGE_STATS_THREAD_DMA_DRIVER, + /* Used for driving AES DMA for random number generation. */ + GCIP_USAGE_STATS_THREAD_GRAPH_DMA_DRIVER, + /* + * The multi-cluster scheduler, this dispatches complex workloads to the major and minor + * TPU clusters. + */ + GCIP_USAGE_STATS_THREAD_MC_SCHEDULER, + /* + * The single-cluster scheduler that dispatches workloads to only a single TPU cluster at + * a time. + */ + GCIP_USAGE_STATS_THREAD_SC_SCHEDULER, + /* + * The thread that dispatches scheduled workloads from the DSP control core directly to the + * DSP cores. + */ + GCIP_USAGE_STATS_THREAD_DSP_CORE_MANAGER, + + /* The number of total threads. Must be located at the end of this enum. */ + GCIP_USAGE_STATS_THREAD_NUM_TYPES, +}; + +/* + * Statistics related to a single thread in firmware. + * Must be kept in sync with firmware `struct ThreadStats`. + */ +struct gcip_usage_stats_thread_stats { + /* The thread in question. */ + enum gcip_usage_stats_thread_stats_thread_id thread_id; + /* Maximum stack usage (in bytes) since last firmware boot. */ + uint32_t max_stack_usage_bytes; + + /* Following fields must not be accessed in version 1. */ + + /* Reserved. */ + uint32_t reserved[2]; +} __packed; + +/* + * Defines different max watermarks we track. + * Must be kept in sync with firmware `enum class MaxWatermarkType`. + */ +enum gcip_usage_stats_max_watermark_type { + /* The number of UCI/VII commands dequeued and not yet responded to. */ + GCIP_USAGE_STATS_MAX_WATERMARK_OUTSTANDING_CMDS, + /* The maximum number of outstanding preempted workloads that must be resumed. */ + GCIP_USAGE_STATS_MAX_WATERMARK_PREEMPTION_DEPTH, + /* + * The longest time in microseconds required to save a cluster-context so that another + * client can run on the same cluster. + */ + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_HW_CONTEXT_SAVE_TIME, + /* + * Maximum time in microseconds spent waiting to hit a scalar fence during hardware + * preemption. + */ + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_SCALAR_FENCE_WAIT_TIME, + /* Maximum time in microseconds spent during the Pipeline::Suspend function. */ + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_SUSPEND_TIME, + + /* The number of total types. Must be located at the end of this enum. */ + GCIP_USAGE_STATS_MAX_WATERMARK_NUM_TYPES, +}; + +/* + * Max watermark. Only reported if it has a value larger than 0. + * Must be kept in sync with firmware `struct MaxWatermark`. + */ +struct gcip_usage_stats_max_watermark { + /* What it counts. */ + enum gcip_usage_stats_max_watermark_type type; + /* Maximum expressed value over the collection interval. */ + uint64_t value; + + /* Following fields must not be accessed in version 1. */ + + /* Reporting component. */ + uint8_t component_id; + /* Reserved. */ + uint8_t reserved[3]; +} __packed; + +/* + * Used to report DVFS frequencies supported by the chip. + * Must be kept in sync with firmware `struct DvfsFrequencyInfo`. + */ +struct gcip_usage_stats_dvfs_frequency_info { + /* An actively supported DVFS Frequency (kHz). */ + uint32_t supported_frequency; + + /* Following fields must not be accessed in lower than V2. */ + + /* Reserved. */ + uint32_t reserved[3]; +} __packed; + +/* + * Header struct in the v1 metric buffer. + * Keep this structure for the compatibility. + */ +struct gcip_usage_stats_header_v1 { + /* Number of metrics being reported. */ + uint32_t num_metrics; + /* Size of each metric struct. */ + uint32_t metric_size; +}; + +/* + * Header struct in the metric buffer. + * Must be kept in sync with firmware `struct UsageTrackerHeader`. + */ +struct gcip_usage_stats_header { + /* Number of bytes in this header. */ + uint16_t header_bytes; + /* Metrics version. */ + uint16_t version; + /* Number of metrics being reported. */ + uint32_t num_metrics; + /* Size of each metric struct. */ + uint32_t metric_size; +}; + +/* + * Encapsulates a single metric reported to the kernel driver. + * Must be kept in sync with firmware `struct UsageTrackerMetric`. + */ +struct gcip_usage_stats_metric { + uint32_t type; + uint8_t reserved[4]; + union { + struct gcip_usage_stats_core_usage core_usage; + struct gcip_usage_stats_component_utilization component_utilization; + struct gcip_usage_stats_counter counter; + struct gcip_usage_stats_thread_stats thread_stats; + struct gcip_usage_stats_max_watermark max_watermark; + struct gcip_usage_stats_dvfs_frequency_info dvfs_frequency_info; + /* The implementation of each metric must fit to 16 bytes. */ + uint8_t impl_reserved[16]; + }; +} __packed; + +/* Operators which are needed while processing usage stats data. */ +struct gcip_usage_stats_ops { + /* + * The callback which sends `GET_USAGE` KCI to get the latest usage stats from the firmware + * synchronously and calls `gcip_usage_stats_process_buffer` function to process them. + * + * This callback is required and will be called when the user tries to read device + * statistics. + * + * Returns KCI response code on success or < 0 on error (typically -ETIMEDOUT). + */ + int (*update_usage_kci)(void *data); + + /* + * Returns the number of default DVFS frequencies. + * If the firmware has never sent `DVFS_FREQUENCY_INFO` metrics, it will use the default + * frequencies which are maintained by the kernel driver. + */ + int (*get_default_dvfs_freqs_num)(void *data); + + /* + * Returns the DVFS frequency of @idx. + * @idx will not exceed the number of default DVFS frequencies which is returned by the + * `get_default_dvfs_freqs_num` operator. + */ + int (*get_default_dvfs_freq)(int idx, void *data); +}; + +/* Structure manages the information of usage stats and device attributes. */ +struct gcip_usage_stats { + /* The version of metrics. */ + enum gcip_usage_stats_version version; + /* The number of subcomponents. (e.g., TPU: clusters, DSP: cores) */ + unsigned int subcomponents; + /* The device to register attributes. */ + struct device *dev; + /* User-data. */ + void *data; + + /* Pointer array of attributes which will be registered to the device. */ + struct attribute **attrs; + /* Attribute group which will contain @attrs. */ + struct attribute_group group; + + /* Operators. */ + const struct gcip_usage_stats_ops *ops; + + /* + * Core usage (per subcomponent). + * Stores stats as an UID to `struct gcip_usage_stats_core_usage_uid_entry` hash table. + * Declare it as a pointer to an array because we have to dynamically allocate multiple + * rows with the fixed column size. + * I.e., (@subcomponents (rows) * BIT(GCIP_USAGE_STATS_UID_HASH_BITS) (cols)) 2d array. + */ + struct hlist_head (*core_usage_htable)[BIT(GCIP_USAGE_STATS_UID_HASH_BITS)]; + /* Component utilization. */ + int32_t component_utilization[GCIP_USAGE_STATS_COMPONENT_UTILIZATION_NUM_TYPES]; + /* + * Counter (per subcomponents). + * Declare it as a pointer to an array because we have to dynamically allocate multiple + * rows with the fixed column size. + * I.e., (@subcomponents (rows) * GCIP_USAGE_STATS_COUNTER_NUM_TYPES (cols)) 2d array. + */ + int64_t (*counter)[GCIP_USAGE_STATS_COUNTER_NUM_TYPES]; + /* Thread statistics. */ + int32_t thread_max_stack_usage[GCIP_USAGE_STATS_THREAD_NUM_TYPES]; + /* + * Max watermark (per subcomponents). + * Declare it as a pointer to an array because we have to dynamically allocate multiple + * rows with the fixed column size. + * I.e., (@subcomponents (rows) * GCIP_USAGE_STATS_MAX_WATERMARK_NUM_TYPES (cols)) 2d + * array. + */ + int64_t (*max_watermark)[GCIP_USAGE_STATS_MAX_WATERMARK_NUM_TYPES]; + /* Protects the statistics above. */ + struct mutex usage_stats_lock; + + /* + * DVFS frequencies that the firmware returns via `DVFS_FREQUENCY_INFO` metric. + * If the firmware has sent `DVFS_FREQUENCY_INFO` metric, it will be used instead of + * getting the default ones from the kernel driver side via @get_default_dvfs_freq callback + * of the `struct gcip_usage_stats_ops`. + */ + uint32_t dvfs_freqs[GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM]; + /* The number of DVFS frequencies. */ + int dvfs_freqs_num; + /* Protects DVFS frequencies. */ + struct mutex dvfs_freqs_lock; +}; + +/* + * Structure which contains information of an attribute to be registered to the device. + * One can directly create an instance, but it is recommneded to use `GCIP_USAGE_STATS_ATTR_*` + * macros instead. A pointer array of this attributes must be passed to the @attrs of + * `struct gcip_usage_stats_args`. + */ +struct gcip_usage_stats_attr { + /* The metric to be collected. */ + enum gcip_usage_stats_metric_type metric; + /* + * The sub-type of @metric to be collected. + * - COMPONENT_UTILIZATION: enum gcip_usage_stats_component_utilization_type + * - COUNTER: enum gcip_usage_stats_counter_type + * - MAX_WATERMARK: enum gcip_usage_stats_max_watermark_type + */ + unsigned int type; + /* + * The 0-based index of subcomponent. (Ignored in V1 metrics.) + * + * One can specify the subcomponent to be read if there are multiple subcomponents. + * + * If this value is `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS`, its show or store function + * will involve in all subcomponents. In case of show, it will print statistics of all + * subcomponents in an array with whitespace separation. Note that according to the Linux + * documentation, one value per one attribute is a rule, but printing multiple same types + * in an array is acceptable. Therefore, use this way only when the printing format is + * simple. In case of store, it will update statistic values (mostly reset to 0) of all + * subcomponents. + * + * Note: when the metric is `CORE_USAGE`, using `GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS` + * is invalid because its printing format is too complicated to print multiple + * subcomponents in one attribute. + */ + int subcomponent; + /* The name of the attribute. */ + const char *name; + /* Permission. It must be one of `GCIP_USAGE_STATS_MODE_*`. */ + umode_t mode; + /* + * User-defined show callback. + * + * Mostly, one will set it as NULL to use the GCIP implementation. + * See the `gcip_usage_stats_alloc_attrs` function to find which function will be used for + * the show function according to the type of metric. + * + * However, if a customized show function is needed, one can pass its own function to this. + * + * It will be used only when @mode has the read permission. + */ + gcip_usage_stats_show_t show; + + /* + * User-defined store callback. + * + * Mostly, one will set it as NULL to use the GCIP implementation. + * See the `gcip_usage_stats_alloc_attrs` function to find which function will be used for + * the store function according to the type of metric. + * + * However, if a customized store function is needed, one can pass its own function to + * this. + * + * It will be used only when @mode has the write permission. + */ + gcip_usage_stats_store_t store; + + /* Following fields must not be touched by the caller. */ + struct device_attribute dev_attr; + struct gcip_usage_stats *ustats; +}; + +/* + * Arguments for `gcip_usage_stats_init`. + * + * `struct gcip_usage_stats` instance will be initialized according to this. + */ +struct gcip_usage_stats_args { + /* The version of metrics. */ + enum gcip_usage_stats_version version; + /* + * The number of subcomponents. (e.g., TPU: clusters, DSP: cores) + * Must be bigger than 0. + */ + unsigned int subcomponents; + /* The device to register attributes. */ + struct device *dev; + /* User-data. */ + void *data; + /* + * Pointer array of attributes. + * This must not be freed before the `gcip_usage_stats_exit` is called. + */ + struct gcip_usage_stats_attr **attrs; + /* The size of @attrs. */ + unsigned int num_attrs; + /* + * Operators. + * See `struct gcip_usage_stats_ops` for the details. + */ + const struct gcip_usage_stats_ops *ops; +}; + +/* + * Initializes @ustats. + * + * @ustats must be cleaned up with the `gcip_usage_stats_exit` function. + */ +int gcip_usage_stats_init(struct gcip_usage_stats *ustats, + const struct gcip_usage_stats_args *args); + +/* Cleans up @ustats which is initialized by the `gcip_usage_stats_init` function. */ +void gcip_usage_stats_exit(struct gcip_usage_stats *ustats); + +/* Processes the buffer which is returned by the firmware via `GET_USAGE` KCI. */ +void gcip_usage_stats_process_buffer(struct gcip_usage_stats *ustats, void *buf); + +#endif /* __GCIP_USAGE_STATS_H__ */ diff --git a/gxp-debug-dump.c b/gxp-debug-dump.c index e0947c1..a63a044 100644 --- a/gxp-debug-dump.c +++ b/gxp-debug-dump.c @@ -582,9 +582,9 @@ static int gxp_handle_debug_dump(struct gxp_dev *gxp, struct gxp_core_dump_header *core_dump_header = &core_dump->core_dump_header[core_id]; struct gxp_core_header *core_header = &core_dump_header->core_header; - int virt_core; int ret = 0; #if HAS_COREDUMP + int virt_core; struct gxp_common_dump *common_dump = mgr->common_dump; int i; int seg_idx = 0; diff --git a/gxp-mcu-fs.c b/gxp-mcu-fs.c index a31026c..6f25618 100644 --- a/gxp-mcu-fs.c +++ b/gxp-mcu-fs.c @@ -89,7 +89,9 @@ gxp_ioctl_uci_response(struct gxp_client *client, down_read(&client->semaphore); - if (!gxp_client_has_available_vd(client, "GXP_MAILBOX_UCI_RESPONSE")) { + if (!client->vd) { + dev_err(client->gxp->dev, + "GXP_MAILBOX_UCI_RESPONSE requires the client allocate a VIRTUAL_DEVICE\n"); ret = -ENODEV; goto out; } diff --git a/gxp-mcu-platform.c b/gxp-mcu-platform.c index 0c4d2aa..203f1d5 100644 --- a/gxp-mcu-platform.c +++ b/gxp-mcu-platform.c @@ -15,13 +15,7 @@ #include "gxp-mcu-platform.h" #include "gxp-mcu.h" #include "gxp-usage-stats.h" - -#define KCI_RETURN_CORE_LIST_MASK 0xFF00 -#define KCI_RETURN_CORE_LIST_SHIFT 8 -#define KCI_RETURN_ERROR_CODE_MASK (BIT(KCI_RETURN_CORE_LIST_SHIFT) - 1u) -#define KCI_RETURN_GET_CORE_LIST(ret) \ - ((KCI_RETURN_CORE_LIST_MASK & (ret)) >> KCI_RETURN_CORE_LIST_SHIFT) -#define KCI_RETURN_GET_ERROR_CODE(ret) (KCI_RETURN_ERROR_CODE_MASK & (ret)) +#include "gxp-vd.h" #if IS_ENABLED(CONFIG_GXP_TEST) char *gxp_work_mode_name = "mcu"; @@ -88,35 +82,6 @@ static int allocate_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd) return 0; } -static void release_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd) -{ - struct gxp_kci *kci = &(gxp_mcu_of(gxp)->kci); - uint core_list; - int ret; - - if (vd->client_id < 0) - return; - - ret = gxp_kci_release_vmbox(kci, vd->client_id); - if (!ret) - goto out; - if (ret > 0 && - KCI_RETURN_GET_ERROR_CODE(ret) == GCIP_KCI_ERROR_ABORTED) { - core_list = KCI_RETURN_GET_CORE_LIST(ret); - dev_err(gxp->dev, - "Firmware failed to gracefully release a VMBox for client %d, core_list=%d", - vd->client_id, core_list); - gxp_vd_invalidate(gxp, vd); - gxp_vd_generate_debug_dump(gxp, vd, core_list); - } else { - dev_err(gxp->dev, - "Failed to request releasing VMBox for client %d: %d", - vd->client_id, ret); - } -out: - vd->client_id = -1; -} - static int gxp_mcu_link_offload_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd, u32 offload_client_id, @@ -136,24 +101,6 @@ static int gxp_mcu_link_offload_vmbox(struct gxp_dev *gxp, return ret; } -static void gxp_mcu_unlink_offload_vmbox(struct gxp_dev *gxp, - struct gxp_virtual_device *vd, - u32 offload_client_id, - u8 offload_chip_type) -{ - struct gxp_kci *kci = &(gxp_mcu_of(gxp)->kci); - int ret; - - ret = gxp_kci_link_unlink_offload_vmbox(kci, vd->client_id, - offload_client_id, - offload_chip_type, false); - if (ret) - dev_err(gxp->dev, - "Failed to unlink offload VMBox for client %d, offload client %u, offload chip type %d: %d", - vd->client_id, offload_client_id, offload_chip_type, - ret); -} - static int gxp_mcu_platform_after_vd_block_ready(struct gxp_dev *gxp, struct gxp_virtual_device *vd) { @@ -177,7 +124,7 @@ static int gxp_mcu_platform_after_vd_block_ready(struct gxp_dev *gxp, return 0; err_release_vmbox: - release_vmbox(gxp, vd); + gxp_vd_release_vmbox(gxp, vd); return ret; } @@ -189,10 +136,7 @@ gxp_mcu_platform_before_vd_block_unready(struct gxp_dev *gxp, return; if (vd->client_id < 0 || vd->mcu_crashed) return; - if (vd->tpu_client_id >= 0) - gxp_mcu_unlink_offload_vmbox(gxp, vd, vd->tpu_client_id, - GCIP_KCI_OFFLOAD_CHIP_TYPE_TPU); - release_vmbox(gxp, vd); + gxp_vd_release_vmbox(gxp, vd); } static int gxp_mcu_pm_after_blk_on(struct gxp_dev *gxp) @@ -260,15 +204,11 @@ static int gxp_mcu_after_map_tpu_mbx_queue(struct gxp_dev *gxp, return 0; } -static void gxp_mcu_before_unmap_tpu_mbx_queue(struct gxp_dev *gxp, - struct gxp_client *client) +static void gxp_mcu_before_unmap_tpu_mbx_queue(struct gxp_dev *gxp, struct gxp_client *client) { struct gxp_virtual_device *vd = client->vd; - if (vd->client_id >= 0 && vd->tpu_client_id >= 0) - gxp_mcu_unlink_offload_vmbox(gxp, vd, vd->tpu_client_id, - GCIP_KCI_OFFLOAD_CHIP_TYPE_TPU); - vd->tpu_client_id = -1; + gxp_vd_unlink_offload_vmbox(gxp, vd, vd->tpu_client_id, GCIP_KCI_OFFLOAD_CHIP_TYPE_TPU); } #endif /* HAS_TPU_EXT */ @@ -34,8 +34,22 @@ #include "gxp-pm.h" #include "gxp-vd.h" +#if GXP_HAS_MCU +#include <gcip/gcip-kci.h> + +#include "gxp-kci.h" +#include "gxp-mcu.h" +#endif + #include <trace/events/gxp.h> +#define KCI_RETURN_CORE_LIST_MASK 0xFF00 +#define KCI_RETURN_CORE_LIST_SHIFT 8 +#define KCI_RETURN_ERROR_CODE_MASK (BIT(KCI_RETURN_CORE_LIST_SHIFT) - 1u) +#define KCI_RETURN_GET_CORE_LIST(ret) \ + ((KCI_RETURN_CORE_LIST_MASK & (ret)) >> KCI_RETURN_CORE_LIST_SHIFT) +#define KCI_RETURN_GET_ERROR_CODE(ret) (KCI_RETURN_ERROR_CODE_MASK & (ret)) + static inline void hold_core_in_reset(struct gxp_dev *gxp, uint core) { gxp_write_32(gxp, GXP_CORE_REG_ETM_PWRCTL(core), @@ -1472,10 +1486,15 @@ void gxp_vd_invalidate_with_client_id(struct gxp_dev *gxp, int client_id, } gxp_vd_invalidate(gxp, client->vd); - gxp_vd_generate_debug_dump(gxp, client->vd, core_list); + /* + * Release @client->semaphore first because the `gxp_vd_generate_debug_dump` function only + * requires holding @gxp->vd_semaphore and holding @client->semaphore will block the client + * calling ioctls for a while as generating debug dump taking long time. + */ + up_write(&client->semaphore); + gxp_vd_generate_debug_dump(gxp, client->vd, core_list); up_write(&gxp->vd_semaphore); - up_write(&client->semaphore); } void gxp_vd_invalidate(struct gxp_dev *gxp, struct gxp_virtual_device *vd) @@ -1486,11 +1505,7 @@ void gxp_vd_invalidate(struct gxp_dev *gxp, struct gxp_virtual_device *vd) vd->client_id); if (vd->state != GXP_VD_UNAVAILABLE) { - if (gxp->mailbox_mgr->release_unconsumed_async_resps) - gxp->mailbox_mgr->release_unconsumed_async_resps(vd); - vd->state = GXP_VD_UNAVAILABLE; - if (vd->invalidate_eventfd) gxp_eventfd_signal(vd->invalidate_eventfd); } else { @@ -1537,3 +1552,55 @@ void gxp_vd_generate_debug_dump(struct gxp_dev *gxp, down_write(&gxp->vd_semaphore); gxp_vd_put(vd); } + +#if GXP_HAS_MCU +void gxp_vd_release_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd) +{ + struct gxp_kci *kci = &(gxp_mcu_of(gxp)->kci); + uint core_list; + int ret; + + if (vd->client_id < 0 || vd->mcu_crashed) + goto out; + + if (vd->tpu_client_id >= 0) + gxp_vd_unlink_offload_vmbox(gxp, vd, vd->tpu_client_id, + GCIP_KCI_OFFLOAD_CHIP_TYPE_TPU); + + ret = gxp_kci_release_vmbox(kci, vd->client_id); + if (!ret) + goto out; + if (ret > 0 && KCI_RETURN_GET_ERROR_CODE(ret) == GCIP_KCI_ERROR_ABORTED) { + core_list = KCI_RETURN_GET_CORE_LIST(ret); + dev_err(gxp->dev, + "Firmware failed to gracefully release a VMBox for client %d, core_list=%d", + vd->client_id, core_list); + gxp_vd_invalidate(gxp, vd); + gxp_vd_generate_debug_dump(gxp, vd, core_list); + } else { + dev_err(gxp->dev, "Failed to request releasing VMBox for client %d: %d", + vd->client_id, ret); + } +out: + vd->client_id = -1; +} + +void gxp_vd_unlink_offload_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd, + u32 offload_client_id, u8 offload_chip_type) +{ + struct gxp_kci *kci = &(gxp_mcu_of(gxp)->kci); + int ret; + + if (vd->client_id < 0 || vd->tpu_client_id < 0 || vd->mcu_crashed) + goto out; + + ret = gxp_kci_link_unlink_offload_vmbox(kci, vd->client_id, offload_client_id, + offload_chip_type, false); + if (ret) + dev_err(gxp->dev, + "Failed to unlink offload VMBox for client %d, offload client %u, offload chip type %d: %d", + vd->client_id, offload_client_id, offload_chip_type, ret); +out: + vd->tpu_client_id = -1; +} +#endif /* GXP_HAS_MCU */ @@ -446,6 +446,37 @@ void gxp_vd_invalidate(struct gxp_dev *gxp, struct gxp_virtual_device *vd); void gxp_vd_generate_debug_dump(struct gxp_dev *gxp, struct gxp_virtual_device *vd, uint core_list); +#if GXP_HAS_MCU +/* + * Releases the vmbox which is allocated to @vd. + * + * This function will call the `RELEASE_VMBOX` KCI and will always set @vd->client_id to -1. If the + * vmbox was linked to the offload vmbox, it will also call the `gxp_vd_unlink_offload_vmbox` + * function first internally. + * + * @gxp: The GXP device to obtain the handler for. + * @vd: The virtual device to release its vmbox. + */ +void gxp_vd_release_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd); + +/* + * Unlinks the linkage of the vmbox of @vd to the offload chip vmbox. + * + * This function will call the `UNLINK_OFFLOAD_VMBOX` KCI to unlink the vmboxes and will always set + * @vd->tpu_client_id to -1. + * + * @gxp: The GXP device to obtain the handler for. + * @vd: The virtual device to unlink vmboxes. + * @offload_client_id: The client ID of the offload chip. + * @offload_chip_type: The type of the offload chip. (See enum gcip_kci_offload_chip_type.) + */ +void gxp_vd_unlink_offload_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd, + u32 offload_client_id, u8 offload_chip_type); +#else /* !GXP_HAS_MCU */ +#define gxp_vd_release_vmbox(...) +#define gxp_vd_unlink_offload_vmbox(...) +#endif /* GXP_HAS_MCU */ + /* * An ID between 0~GXP_NUM_CORES-1 and is unique to each VD. * Only used in direct mode. |